diff --git a/.github/workflows/blog-prose.yml b/.github/workflows/blog-prose.yml index b287e14b353..a0da186cfa7 100644 --- a/.github/workflows/blog-prose.yml +++ b/.github/workflows/blog-prose.yml @@ -131,6 +131,11 @@ jobs: cd scripts npm init -y 2>/dev/null || true npm install playwright + # Playwright shells out to apt for the OS-level deps, so the runner + # image's third-party sources have to be pruned FIRST -- a broken + # vendor mirror makes its apt-get fail as a whole and it reports + # only "Failed to install browser dependencies". + bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-update.sh" npx playwright install-deps chromium npx playwright install chromium diff --git a/.github/workflows/blog-syndication.yml b/.github/workflows/blog-syndication.yml index 9e0a00213fa..762b852cdbb 100644 --- a/.github/workflows/blog-syndication.yml +++ b/.github/workflows/blog-syndication.yml @@ -79,6 +79,11 @@ jobs: if: ${{ steps.browser_creds.outputs.any_configured == 'true' }} run: | set -euo pipefail + # Playwright shells out to apt for the OS-level deps, so the runner + # image's third-party sources have to be pruned FIRST -- a broken + # vendor mirror makes its apt-get fail as a whole and it reports + # only "Failed to install browser dependencies". + bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-update.sh" pip install playwright playwright install --with-deps chromium diff --git a/.github/workflows/developer-guide-docs.yml b/.github/workflows/developer-guide-docs.yml index 8bb420da735..ecaf1d51175 100644 --- a/.github/workflows/developer-guide-docs.yml +++ b/.github/workflows/developer-guide-docs.yml @@ -133,7 +133,7 @@ jobs: run: | set -euo pipefail xvfb-run -a mvn -B -ntp -f maven/pom.xml \ - -pl core,javase,android,css-compiler,codenameone-maven-plugin \ + -pl core,javase,android,css-compiler,codenameone-maven-plugin,backend \ -am install \ -Plocal-dev-javase \ -DskipTests \ diff --git a/.github/workflows/port-status-nightly.yml b/.github/workflows/port-status-nightly.yml index 56036a4b261..7cd0bdc75e3 100644 --- a/.github/workflows/port-status-nightly.yml +++ b/.github/workflows/port-status-nightly.yml @@ -82,6 +82,11 @@ jobs: run: | npm init -y 2>/dev/null || true npm install playwright + # Playwright shells out to apt for the OS-level deps, so the runner + # image's third-party sources have to be pruned FIRST -- a broken + # vendor mirror makes its apt-get fail as a whole and it reports + # only "Failed to install browser dependencies". + bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-update.sh" npx playwright install --with-deps "${{ matrix.browser }}" - name: Run lifecycle validation env: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cf0e7d99fb8..c677824c720 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,6 +27,7 @@ on: - 'scripts/ci/retry.sh' - 'scripts/ci/apt-get-update.sh' - 'scripts/ci/apt-get-install.sh' + - 'scripts/ci/tests/apt-get-update-test.sh' # The build hint gates are run from this workflow and nowhere else, and one # of them holds an empty baseline. Ignoring the whole directory meant a # change that breaks a gate, or that adds a line to the baseline, could @@ -78,6 +79,7 @@ on: - 'scripts/ci/retry.sh' - 'scripts/ci/apt-get-update.sh' - 'scripts/ci/apt-get-install.sh' + - 'scripts/ci/tests/apt-get-update-test.sh' # The build hint gates are run from this workflow and nowhere else, and one # of them holds an empty baseline. Ignoring the whole directory meant a # change that breaks a gate, or that adds a line to the baseline, could @@ -469,6 +471,12 @@ jobs: - name: Check build hint catalog if: ${{ matrix.java-version == 8 }} run: scripts/check-build-hint-catalog.sh + - name: Check the apt source prune keeps Ubuntu's own sources + if: ${{ matrix.java-version == 8 }} + # Cheap, and the rule it covers is only ever exercised on a runner whose + # apt is already broken -- so without this it would be tested by a red + # build and nothing else. + run: bash scripts/ci/tests/apt-get-update-test.sh - name: Check the build hint data file can be rendered if: ${{ matrix.java-version == 8 }} # Not a drift check: nothing is committed to drift from, because every diff --git a/.github/workflows/scripts-javascript.yml b/.github/workflows/scripts-javascript.yml index 1b38763644b..4f999727ff0 100644 --- a/.github/workflows/scripts-javascript.yml +++ b/.github/workflows/scripts-javascript.yml @@ -210,6 +210,11 @@ jobs: npm init -y 2>/dev/null || true npm install playwright # OS-level deps (apt packages) aren't cached so always install them. + # Playwright shells out to apt for the OS-level deps, so the runner + # image's third-party sources have to be pruned FIRST -- a broken + # vendor mirror makes its apt-get fail as a whole and it reports + # only "Failed to install browser dependencies". + bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-update.sh" npx playwright install-deps chromium # `npm install playwright` resolves the floating version, so the # cached browser binary can drift away from what the freshly diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java index 7c81f0785eb..b4411c4fa8e 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidImplementation.java @@ -9912,8 +9912,18 @@ public void onReceive(Context ctx, Intent intent) { try { appCtx.unregisterReceiver(this); } catch (Throwable ignore) {} String pkg = null; try { - android.content.ComponentName cn = intent.getParcelableExtra(Intent.EXTRA_CHOSEN_COMPONENT); - if (cn != null) pkg = cn.getPackageName(); + // Taken as a Parcelable and tested, rather than assigned straight + // to ComponentName: that assignment compiles to a CHECKCAST whose + // failure this catch would have to handle, and the extra is + // whatever the SENDING application chose to put there, so the + // failure is not hypothetical. (The cast-semantics gate no longer + // scans this port, since ParparVM does not translate it -- this + // stands on its own terms.) + android.os.Parcelable chosen = + intent.getParcelableExtra(Intent.EXTRA_CHOSEN_COMPONENT); + if (chosen instanceof android.content.ComponentName) { + pkg = ((android.content.ComponentName) chosen).getPackageName(); + } } catch (Throwable ignore) {} listener.onResult(com.codename1.share.ShareResult.sharedTo(pkg)); } diff --git a/docs/demos/backend/pom.xml b/docs/demos/backend/pom.xml new file mode 100644 index 00000000000..74f7cea27de --- /dev/null +++ b/docs/demos/backend/pom.xml @@ -0,0 +1,36 @@ + + + 4.0.0 + + com.codenameone.developerguide + democode + 1.0-SNAPSHOT + + backendsnippets + backendsnippets + + + + com.codenameone + codenameone-backend + ${cn1.version} + + + + com.codenameone + codenameone-core + ${cn1.version} + + + diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java new file mode 100644 index 00000000000..7e58bf071b9 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import com.codename1.backend.Database; +import java.io.IOException; +import java.util.List; + +/** The Backend chapter's database examples, compiled so they cannot drift. */ +public final class DatabaseSnippets { + + private DatabaseSnippets() { + } + + public static List open() throws IOException { +// tag::backend-database[] +Database db = Database.open(System.getenv("DATABASE_URL")); // or ":memory:" +db.execute("CREATE TABLE IF NOT EXISTS note (id INTEGER PRIMARY KEY, body TEXT)", + null); + +List rows = db.query("SELECT id, body FROM note WHERE id > ?", + new Object[] { Integer.valueOf(10) }); +// end::backend-database[] + return rows; + } +} diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/Note.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/Note.java new file mode 100644 index 00000000000..009b4e8c340 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/Note.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +/** The data transfer object both ends of {@link NotesApi} share. */ +public class Note { + public long id; + public String body; + + public Note() { + } +} diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/Notes.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/Notes.java new file mode 100644 index 00000000000..a5c1f9edbc6 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/Notes.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import com.codename1.backend.annotations.DeleteMapping; +import com.codename1.backend.annotations.GetMapping; +import com.codename1.backend.annotations.PathVariable; +import com.codename1.backend.annotations.PostMapping; +import com.codename1.backend.annotations.RequestBody; +import com.codename1.backend.annotations.RequestMapping; +import com.codename1.backend.annotations.RequestParam; +import com.codename1.backend.annotations.ResponseStatus; +import com.codename1.backend.annotations.RestController; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +// tag::backend-first-server[] +@RestController +@RequestMapping("/notes") +public class Notes { + private final Map store = new ConcurrentHashMap(); + private final AtomicLong nextId = new AtomicLong(1); + + @GetMapping("/healthz") + public String health() { + return "ok"; + } + + @GetMapping("/{id}") + public Map read(@PathVariable("id") long id) { + return store.get(Long.valueOf(id)); // null becomes a 404 + } + + @GetMapping + public List list(@RequestParam(value = "limit", defaultValue = "20") int limit) { + List page = new ArrayList(); + for (Map note : store.values()) { + if (page.size() >= limit) { + break; + } + page.add(note); + } + return page; + } + + @PostMapping + @ResponseStatus(201) + public Map create(@RequestBody Map note) { + long id = nextId.getAndIncrement(); + Map stored = new LinkedHashMap(note); + stored.put("id", Long.valueOf(id)); + store.put(Long.valueOf(id), stored); + return stored; + } + + @DeleteMapping("/{id}") + @ResponseStatus(204) + public void delete(@PathVariable("id") long id) { + store.remove(Long.valueOf(id)); + } +} +// end::backend-first-server[] diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesApi.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesApi.java new file mode 100644 index 00000000000..a8bab066763 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesApi.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import com.codename1.annotations.rest.Body; +import com.codename1.annotations.rest.GET; +import com.codename1.annotations.rest.POST; +import com.codename1.annotations.rest.Path; +import com.codename1.annotations.rest.RestClient; +import com.codename1.io.rest.Response; +import com.codename1.util.OnComplete; + +// tag::backend-contract[] +@RestClient +public interface NotesApi { + @GET("/notes/{id}") + void note(@Path("id") String id, OnComplete> callback); + + @POST("/notes") + void create(@Body Note note, OnComplete> callback); +} +// end::backend-contract[] diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesApiServer.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesApiServer.java new file mode 100644 index 00000000000..53b94ec215d --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesApiServer.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +/** + * What the build emits from {@link NotesApi} for the server side. + * + * Reproduced here rather than generated: the generator runs after this module + * compiles, and the point of these files is that the guide's examples compile at + * all. A real project never writes this -- it comes out of the same annotated + * contract the app's client comes out of, method for method. + */ +interface NotesApiServer { + Note note(String id) throws Exception; + + Note create(Note note) throws Exception; +} diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesEndpoint.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesEndpoint.java new file mode 100644 index 00000000000..fd79b067cac --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesEndpoint.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +// tag::backend-contract-server[] +public class NotesEndpoint implements NotesApiServer { + private final Map notes = new ConcurrentHashMap(); + + public Note note(String id) { // no callback: this IS the server + return notes.get(id); + } + + public Note create(Note note) { + notes.put(String.valueOf(note.id), note); + return note; + } +} +// end::backend-contract-server[] diff --git a/docs/demos/pom.xml b/docs/demos/pom.xml index 0bc4923fbd4..c7200018e26 100644 --- a/docs/demos/pom.xml +++ b/docs/demos/pom.xml @@ -17,6 +17,7 @@ common +backend 8.0-SNAPSHOT diff --git a/docs/developer-guide/Backend.asciidoc b/docs/developer-guide/Backend.asciidoc new file mode 100644 index 00000000000..dea7664dd5f --- /dev/null +++ b/docs/developer-guide/Backend.asciidoc @@ -0,0 +1,340 @@ +== Server-side backend + +The backend compiles a Java HTTP handler into a native executable. It uses the +same ParparVM pipeline as the iOS, Windows and Linux ports: javac produces +bytecode, ParparVM turns that into C, and clang compiles the C together with the +runtime into one binary. Nothing interprets anything at run time, and there is no +JVM underneath. + +The measurements in this chapter come from `vm/backend/benchmarks` on two pinned +cores against 64 connections, and the harness is in the repository so you can +disagree with them. + +.The build pipeline and the request path +image::img/backend-architecture.svg[Java compiled to C to a native binary, and a request travelling through a host thread and a virtual thread to the handler,scaledwidth=95%] + +=== What this doesn't replace + +Spring Boot, Quarkus, Micronaut and Jakarta EE aren't the competition here. +They carry dependency injection, an ORM, declarative transactions, a security +stack and two decades of operational knowledge, and none of that exists in this +runtime. If you are running a Spring service today and it works, this chapter is +not asking you to move it. + +What those frameworks assume is a JVM, and that paying for one is reasonable. For +most server work that assumption holds. This exists for the work where it fails. + +=== Why it exists + +A JVM charges you twice. It charges start-up on every cold process, and it +charges a baseline heap for as long as the process lives. Both are fine when a +service runs for weeks and handles millions of requests. Neither is fine when the +process exits after 200 milliseconds, when you are billed per invocation, or when +the instance is capped at 128 MB. + +That describes a specific and growing slice of server work: serverless functions, +sidecars, edge workers, webhook receivers, small always-on services. Java is thin +on the ground there, and the reason is arithmetic rather than taste. Teams +therefore reach for Go or Node, and a Java shop that does this ends up with two +languages, two toolchains, and two definitions of every object that crosses the +wire. + +The point of the backend is to remove the reason to leave Java for that slice. It +doesn't try to take work the JVM already does well. + +=== A first server + +The archetype and the initializr both generate a `backend` module beside the +client ones: + +---- +myapp/ + common/ shared app code + javase/ desktop build + ios/ iOS build + android/ Android build + backend/ the server + pom.xml + src/main/java/com/example/myapp/Notes.java +---- + +A server is a class with routes on it. The annotations are Spring's, under +Codename One's package names, so this reads the same way to anyone who has +written a Spring controller: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/Notes.java[tag=backend-first-server,indent=0] +---- + +There is no `main` here, and that's the point. Every server opens with the same +twenty lines -- start a listener, install a shutdown handler, wait on it -- and +getting any of them wrong produces one that leaks connections on SIGTERM, or one +that ends the moment `main` returns and says nothing about why. The build writes +those lines, from the controllers it finds. + +It also writes the router. The obvious hand-written form, +`if ("/healthz".equals(request.getTarget()))`, builds a String for the target, +hashes it and compares it -- for every route it tries before the one that matches, +on every request -- and breaks as soon as a client appends `?probe=1`. The +generated router holds each route as a `byte[]` and asks the request whether its +path bytes are those bytes, so a route with no path variables allocates nothing at +all and a query string can't break it. + +The return value decides the response. A `String` is sent as text, anything else as +JSON, `null` is a 404, and `@ResponseStatus` sets the code for the cases where 200 +isn't it. A handler that needs something this doesn't model takes the +`HttpServer.Request` itself and answers exactly as it would have before. + +Two commands matter: + +---- +# the property is not optional: the backend module lives in a profile, and +# without it Maven cannot see it in the reactor at all +mvn -pl backend -Dcodename1.platform=backend cn1:backend # run it on this JVM +mvn -pl backend -Dcodename1.platform=backend cn1:backend-package # build the native binary +---- + +The first is the development loop. Both run the same protocol code -- there is one +copy of `HttpServer`, and only the layer underneath it differs -- so behaviour +can't drift between the loop you develop in and the binary you ship. The local +run doesn't terminate TLS, by design, and therefore doesn't serve HTTP/2: +both refuse with a message that says so, because a second TLS implementation +would have its own bugs rather than production's. + +=== Virtual threads and the request loop + +The concurrency model is the part most worth understanding, because it's what +makes a handler that blocks acceptable. + +Every connection gets a virtual thread. Not a pooled worker that a connection +borrows for one request -- a stack that belongs to that connection until it +closes. The handler can block on a socket read, a database round trip or a file, +and it costs a parked stack rather than an OS thread. + +Host threads run those virtual threads, and there is one host per core. A +descriptor is registered in exactly one host's epoll set, so it can only ever be +reported to that host, and only that host ever touches its virtual thread. That +affinity is why the scheduler needs no locks on the hot path. + +The loop is: the host polls, a descriptor becomes readable, the host resumes that +connection's virtual thread, the handler runs until it needs bytes that haven't +arrived, and it parks. Parking returns control to the host, which polls again. +When the handler finishes a response the descriptor stays armed, so the next +request on that connection costs no system call to set up. + +Host count follows cores rather than the `workers` argument. On two pinned cores, +16 hosts served 117 requests where 2 hosts served 257,297: past one host per core +they compete for the cores the server needs. In this mode `workers` stops meaning +"requests in flight," because the virtual threads supply that. + +=== Talking to a database + +`Database.open` takes a SQLite path or a PostgreSQL or MySQL URL, and the rows +come back as the same Java types either way: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/DatabaseSnippets.java[tag=backend-database,indent=0] +---- + +There is no JDBC driver involved. SQLite is linked into the binary, and the +PostgreSQL and MySQL clients speak their wire protocols directly. + +`DbPool` pools SQLite connections, and only those: it opens them from a file +path. There is no pool for the server engines yet. A `Database` over PostgreSQL +or MySQL owns one wire connection and doesn't serialize access to it, so two +handlers sharing one interleave their prepared-statement exchanges on the same +socket. Give each request its own connection until there is a pool for them. + +=== Sharing the contract with the app + +This is where having the same language on both ends stops being a slogan. An +interface annotated for the REST client generates the app's client: + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesApi.java[tag=backend-contract,indent=0] +---- + +Building the backend module with `-Dcn1.restServer=true` generates two more types +from that same interface: `NotesApiServer`, a synchronous interface the backend +implements, and `NotesApiDispatcher`, which routes a method, path and body to it +and binds the path and query parameters. + +[source,java] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/NotesEndpoint.java[tag=backend-contract-server,indent=0] +---- + +The client's methods are asynchronous because a UI can't block; the server's +methods are synchronous because a handler has nothing to call back into. One declaration +produces both shapes, which is what gRPC does and for the same reason. + +The payoff is that changing the contract breaks the build on whichever side did +not follow it, instead of producing a response the app fails to parse in the +field. The data transfer objects are shared rather than transcribed, and their +codecs are generated on both sides, so there is no handwritten mapping layer to +drift. + +The server half is off by default. Every existing project carries these +interfaces for its client alone, and generating server classes into those builds +would grow them for nothing. + +=== What it costs + +Same handler, three ways, plus Go for an outside reference. Two pinned cores, 64 +connections, medians of three interleaved runs on the plaintext route: + +[cols="2,1,1,1,1"] +|=== +| Runtime | Requests/sec | p50 | p99 | Cold start + +| Codename One native, musl +| 595,610 +| 0.090 ms +| 0.249 ms +| 0.77 ms + +| Codename One native, glibc +| 547,761 +| 0.065 ms +| 4.06 ms +| 2.88 ms + +| Go, fasthttp +| 496,293 +| 0.104 ms +| 2.63 ms +| 2.39 ms + +| The same handler on the JVM +| 187,745 +| 0.260 ms +| 1.60 ms +| 82.5 ms +|=== + +Cold start is the interesting column. It's measured from process spawn to the +first accepted connection, and the static binary reaches it in under a +millisecond -- about a hundred times faster than the same code on a JVM, and +three times faster than Go. That number is the whole serverless argument. + +Throughput is the least interesting one. Beating a tuned Go server by a fifth on +a microbenchmark isn't a reason to move a service; it's only evidence that the +translation doesn't cost you anything. + +.Latency at the median against the 99th percentile +image::img/backend-latency-slope.svg[A slope chart showing that the musl build's tail stays close to its median while the others fan out,scaledwidth=90%] + +The slope chart is the one that matters. Every runtime here has a similar +median. What differs is the distance to the 99th percentile, and that distance is +garbage collection. With the response pooled the plaintext route allocates about +0.1 bytes per request, the collector never runs, and the tail stays at 2.8 times +the median. Give the same server a handler that allocates a map per request and +its tail goes to 80 ms, because the collector shares the cores with the server. + +The honest rule is that the tail follows your allocation rate, not the runtime +badge. The runtime gives you the tools to allocate nothing on the hot path; it +doesn't do it for you. + +==== Memory and size + +[cols="2,1,1"] +|=== +| Runtime | Binary or artifact | Resident under load + +| Codename One native, musl +| 7.95 MB static +| 10-40 MB + +| Codename One native, glibc +| 3.19 MB dynamic +| 14 MB + +| Go, fasthttp +| 5.63 MB static +| 6.3 MB + +| The same handler on the JVM +| 0.13 MB jar, plus a JRE +| 190 MB +|=== + +The JVM row is the same handler and the same protocol code. Everything it costs +above the native rows is the runtime underneath it. + +The resident figures move around more than the latency ones, because the +collector keeps a pool of pages sized to the busiest moment the process has seen +and gives them back gradually. At rest the native builds sit near 3 MB. + +==== musl or glibc + +Both are supported, and they aren't equivalent. The static musl build starts +faster and has a far shorter tail. The glibc build has a better median, because +its allocator is better under contention, and a smaller binary, because it links +the system libraries instead of carrying them. + +The reason to pick musl isn't the median. It's that the artifact is one file +with nothing underneath it, which is what makes the container the binary and the +cold start a process exec. + +==== What isn't measured here + +GraalVM is missing from these tables on purpose. A fair comparison would have to +run the same handler, and this handler can't run on GraalVM: the runtime's +native methods are ParparVM's, so comparing would mean benchmarking a different +server written against a different framework and reporting it as though the +toolchains had been compared. That's a benchmark worth building, and it isn't +this one. + +=== Deploying it + +The musl build is a single static file, so the container that carries it can be +empty: + +---- +FROM scratch +COPY bench-linux-musl-arm64 /server +ENTRYPOINT ["/server"] +---- + +There is no base image to patch, because there is no base image. `cn1:backend-package` +builds for the machine it runs on; the cross-compiled targets +(`musl-x86_64`, `musl-arm64`, `glibc-x86_64`, `glibc-arm64`) are produced by +`vm/backend/package.sh` in the Codename One repository, which drives one builder +image per target. + +For AWS Lambda, `LambdaRuntime` implements the custom runtime loop. The Lambda +Runtime API is a plaintext poll over loopback, so it needs no listening socket and +no TLS, and what it does need is exactly what a translated binary is good at. + +=== Limits worth knowing + +* The class library is the Codename One runtime, not Java SE. A server dependency + that assumes the full JDK won't translate, and Maven Central isn't the + ecosystem this draws on. +* Routing is the whole of the framework. There is no dependency injection, no + configuration model, no aspect layer, no starter ecosystem; validation and error + mapping are written by hand or generated from the REST contract. +* The packaging goal compiles Java. It recompiles the module's sources against the + backend's class library instead of reusing the jar Maven built, which is what + keeps a server off classes the runtime doesn't have, and is also why Kotlin is + not wired into this path yet even though the client ports support it. +* A virtual thread parks on the socket it's SERVING, and not on any other. An + outbound read -- a database query, an HTTP call to another service -- blocks the + host thread that's running it, because only the server's own descriptors are + registered with a poller that can wake them. With one host per core, that many + concurrent slow outbound calls occupy every host and other connections wait. + Servers that mostly compute and serve get the full benefit; servers whose + handlers spend their time waiting on a database should size for that, or run the + thread pool with `CN1_HTTP_POLL_MODE=0`. +* TLS runs on the thread pool, whatever the poll mode says. The TLS layer can't + park a read yet, and a blocking read on a virtual thread holds its host for the + duration, so one idle TLS client per core would occupy every one of them. The + server says so at startup when it makes that choice. +* Native builds target Linux. Development happens anywhere a JVM runs. + +The reasons to choose this are cold start, footprint, deployment shape, and one +language across the app and its server. If none of those matter for the service in +front of you, use a JVM framework. diff --git a/docs/developer-guide/developer-guide.asciidoc b/docs/developer-guide/developer-guide.asciidoc index 1305f7cca88..32fec7f8d26 100644 --- a/docs/developer-guide/developer-guide.asciidoc +++ b/docs/developer-guide/developer-guide.asciidoc @@ -119,6 +119,10 @@ include::Video-IO.asciidoc[] include::Video-Capture-Constraints.asciidoc[] += Server side + +include::Backend.asciidoc[] + = Device and platform services include::Push-Notifications.asciidoc[] diff --git a/docs/developer-guide/img/backend-architecture.svg b/docs/developer-guide/img/backend-architecture.svg new file mode 100644 index 00000000000..e438560dbe6 --- /dev/null +++ b/docs/developer-guide/img/backend-architecture.svg @@ -0,0 +1,54 @@ + + + + How a request is served + Build + + Your Java + handler + runtime + + + Bytecode + javac + + + C sources + ParparVM + + + Object code + clang -O3 + + + One binary + no JVM + Runtime + + Connection + accepted once + + + Host thread + one per core, epoll + + + Virtual thread + one per connection + + + Your handler + plain Java + + parks on I/O; the host resumes it when the socket is ready + In the binary + + HTTP/1.1 + HTTP/2 + + TLS + + SQLite / PostgreSQL / MySQL + + JSON, JWT, static files + No JVM, no interpreter, no application server. The process is the binary, and the container can hold nothing else. + Threads are cheap because a parked virtual thread is a stack and a register set, not an OS thread. + diff --git a/docs/developer-guide/img/backend-latency-slope.svg b/docs/developer-guide/img/backend-latency-slope.svg new file mode 100644 index 00000000000..d0716820447 --- /dev/null +++ b/docs/developer-guide/img/backend-latency-slope.svg @@ -0,0 +1,42 @@ + + + Latency at the median and the 99th percentile + two pinned cores, 64 connections, plaintext route. Logarithmic scale: a steeper line is a longer tail. + + 50 us + + 100 us + + 200 us + + 500 us + + 1 ms + + 2 ms + + 5 ms + p50 + p99 + + + + Codename One native (musl) + 249 us + + + + Codename One native (glibc) + 4.1 ms + + + + Go fasthttp + 2.6 ms + + + + Same handler on the JVM + 1.6 ms + The flat line is the point: with the response pooled the collector never runs, so the tail stays near the median. + diff --git a/docs/developer-guide/languagetool-accept.txt b/docs/developer-guide/languagetool-accept.txt index 5b9d2aeaa75..40bd88ca353 100644 --- a/docs/developer-guide/languagetool-accept.txt +++ b/docs/developer-guide/languagetool-accept.txt @@ -743,3 +743,18 @@ Bodymovin # A point in an animation where a property's value is pinned, with the values in # between interpolated. The universal term across every animation toolchain. keyframes? + +# ----------------------------------------------------------------------------- +# Server-side backend (Backend.asciidoc). +# ----------------------------------------------------------------------------- +# Two of the JVM server frameworks the backend chapter positions itself against. +# Product names, so the dictionary has neither. +Quarkus +Micronaut +# A service that rescales images, in the chapter's list of small edge-shaped +# services the backend suits. Ordinary English formation the dictionary lacks. +resizer +# The Linux readiness-notification interface the request loop is built on. +epoll +# A benchmark of one narrow operation, as against a whole application. +microbenchmark diff --git a/maven/backend/pom.xml b/maven/backend/pom.xml new file mode 100644 index 00000000000..c1fbaf2cad3 --- /dev/null +++ b/maven/backend/pom.xml @@ -0,0 +1,132 @@ + + + + + + com.codenameone + codenameone + 8.0-SNAPSHOT + + 4.0.0 + + codenameone-backend + jar + Codename One Backend Runtime + + + UTF-8 + 1.8 + 1.8 + ${project.basedir}/../../vm/backend + + + + + + org.xerial + sqlite-jdbc + 3.46.1.0 + true + + + + + ${backend.dir}/src + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-javase-implementation + generate-sources + + add-source + + + + ${backend.dir}/impl/javase + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + parparvm-sources-jar + package + + run + + + + + + + + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-parparvm-sources + package + + attach-artifact + + + + + ${project.build.directory}/${project.build.finalName}-parparvm-sources.jar + jar + parparvm-sources + + + + + + + + + diff --git a/maven/cn1app-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml b/maven/cn1app-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml index fb723f58259..259093456c5 100644 --- a/maven/cn1app-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml +++ b/maven/cn1app-archetype/src/main/resources/META-INF/maven/archetype-metadata.xml @@ -261,5 +261,15 @@ + + + + src/main/java + + **/*.java + + + + diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/backend/pom.xml b/maven/cn1app-archetype/src/main/resources/archetype-resources/backend/pom.xml new file mode 100644 index 00000000000..352afcb18e8 --- /dev/null +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/backend/pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + + ${groupId} + ${rootArtifactId} + ${version} + + ${groupId} + ${artifactId} + ${version} + jar + + ${artifactId} + + + + + com.codenameone + codenameone-backend + ${cn1.version} + + + + org.xerial + sqlite-jdbc + 3.46.1.0 + runtime + + + + + + + com.codenameone + codenameone-maven-plugin + ${cn1.plugin.version} + + + + cn1-process-annotations + process-classes + + + process-annotations + + + + + + + diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/backend/src/main/java/Api.java b/maven/cn1app-archetype/src/main/resources/archetype-resources/backend/src/main/java/Api.java new file mode 100644 index 00000000000..9a2ed41349d --- /dev/null +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/backend/src/main/java/Api.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package ${package}; + +import com.codename1.backend.annotations.GetMapping; +import com.codename1.backend.annotations.RequestParam; +import com.codename1.backend.annotations.RestController; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The server side of this app. + * + * Routes are methods. The annotations are Spring's, under Codename One's package + * names, and the build turns them into a router that matches on the request's own + * bytes plus the `main` that serves them -- so there is no server lifecycle to + * write here and no route table to keep in step by hand. + * + * While developing, run it with + * + * mvn -pl backend -Dcodename1.platform=backend cn1:backend + * + * which starts on this JVM in a couple of seconds against the minute and a half a + * native build takes, and whose protocol layer is the same source that ships. The + * property is not optional: the backend module lives in a profile, so without it + * Maven cannot see it in the reactor. Package it with + * + * mvn -pl backend -Dcodename1.platform=backend cn1:backend-package + * + * to get a single native binary with no JVM to install beneath it. + * + * The local run deliberately does not terminate TLS, and therefore does not serve + * HTTP/2. Build the binary when those are what you need to exercise. + */ +@RestController +public class Api { + + /** What a load balancer polls. A String answer is sent as text. */ + @GetMapping("/healthz") + public String health() { + return "ok"; + } + + /** Anything that is not a String is sent as JSON. */ + @GetMapping("/echo") + public Map echo(@RequestParam(value = "say", defaultValue = "hello") String say) { + Map out = new LinkedHashMap(); + out.put("say", say); + return out; + } +} diff --git a/maven/cn1app-archetype/src/main/resources/archetype-resources/pom.xml b/maven/cn1app-archetype/src/main/resources/archetype-resources/pom.xml index c659deb855e..14d8b3569de 100644 --- a/maven/cn1app-archetype/src/main/resources/archetype-resources/pom.xml +++ b/maven/cn1app-archetype/src/main/resources/archetype-resources/pom.xml @@ -218,6 +218,25 @@ linux + + + backend + + + codename1.platform + backend + + + + backend + + android diff --git a/maven/codenameone-maven-plugin/pom.xml b/maven/codenameone-maven-plugin/pom.xml index 6a2f6b03bde..ce7b1a5f659 100644 --- a/maven/codenameone-maven-plugin/pom.xml +++ b/maven/codenameone-maven-plugin/pom.xml @@ -55,6 +55,18 @@ jdom2 2.0.6.1 + + + com.codenameone + codenameone-backend + ${project.version} + test + junit junit diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/BackendPackageMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/BackendPackageMojo.java new file mode 100644 index 00000000000..843374428f1 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/BackendPackageMojo.java @@ -0,0 +1,834 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import org.apache.maven.artifact.Artifact; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.Component; +import org.apache.maven.plugins.annotations.Execute; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.project.MavenProject; +import org.apache.maven.repository.RepositorySystem; +import org.apache.maven.artifact.resolver.ArtifactResolutionRequest; +import org.apache.maven.artifact.resolver.ArtifactResolutionResult; +import org.apache.maven.artifact.repository.ArtifactRepository; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import com.codename1.maven.annotations.AnnotatedClass; +import com.codename1.maven.annotations.ClassScanner; +import com.codename1.maven.annotations.ProcessingException; +import com.codename1.maven.annotations.ProcessorContext; +import com.codename1.maven.processors.RestControllerAnnotationProcessor; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Translates a backend module to C and compiles it to a native binary: + * `mvn cn1:backend-package`. + * + * The counterpart to {@link BackendRunMojo}. That one runs the module on this JVM + * in a couple of seconds and is what a developer uses; this one produces the + * artifact that deploys -- a single executable with no runtime to install, which + * is what makes a scratch container the size of the binary and a Lambda cold start + * a process exec. + * + * What it does, in order: + * + * 1. Compiles the module's sources together with the backend runtime's SHARED and + * PARPARVM halves against the ParparVM JavaAPI as the BOOTCLASSPATH. That last + * part is the important one: the bootclasspath IS the server-safe surface, so a + * reference to something the translated runtime does not have fails here, in + * the IDE and in the build, rather than at link time or in production. + * 2. Runs the translator over the result, with the runtime's C sources already in + * the source root -- they have to be there BEFORE it runs, because a native's + * Java method is kept alive by its C symbol being present. + * 3. Compiles the generated C, either with the host compiler or, for a named + * Linux target, in a container. + * + * The compiler flags are not negotiable and are documented at the call site: + * generated C relies on wrapping arithmetic, and clang -O3 miscompiles it without + * them. + */ +// Forks the lifecycle up to compile first, so `mvn cn1:backend-package` on its own does +// the obvious thing on a clean checkout instead of failing on an empty +// target/classes. +@Execute(phase = LifecyclePhase.COMPILE) +@Mojo(name = "backend-package", requiresDependencyResolution = ResolutionScope.COMPILE) +public class BackendPackageMojo extends AbstractMojo { + + @Parameter(defaultValue = "${project}", readonly = true, required = true) + private MavenProject project; + + @Parameter(defaultValue = "${localRepository}", readonly = true, required = true) + private ArtifactRepository localRepository; + + @Component + private RepositorySystem repositorySystem; + + /** + * The class whose main() becomes the program's entry point. + * + * Optional. A module whose server is written as `@RestController` classes has no + * main of its own -- one is generated from them -- and naming a class that does + * not exist is worse than leaving this out. + */ + @Parameter(property = "cn1.backend.mainClass") + private String mainClass; + + /** + * Where the binary goes. Defaults to target/<artifactId>. + */ + @Parameter(property = "cn1.backend.output") + private File output; + + /** + * A Linux deployment target -- musl-x86_64, musl-arm64, glibc-x86_64, + * glibc-arm64 -- built in a container. Omitted, it compiles for this machine + * with the host compiler, which is what a developer wants and what CI checks. + */ + @Parameter(property = "cn1.backend.target") + private String target; + + /** A JDK 8, which is what the translator's front end requires. */ + @Parameter(property = "cn1.backend.jdk8", defaultValue = "${env.JDK_8_HOME}") + private String jdk8Home; + + /** Extra flags for the C compiler. */ + @Parameter(property = "cn1.backend.cflags") + private String cflags; + + /** + * Whether to link the bundled SQLite engine. Off saves about 2MB in a service + * that talks to PostgreSQL or MySQL instead, which speak their wire protocols + * with no engine linked at all. + */ + @Parameter(property = "cn1.backend.sqlite", defaultValue = "true") + private boolean sqlite; + + /** + * Whether a failed cast throws. + * + * On by default here and off for app targets, which is the one place the + * server build departs from the mobile one deliberately: on a phone a bad cast + * costs one user a crash, and on a server the object with the wrong type + * arrived from the network, so reading its fields as another type kills every + * connection the process was serving. + */ + @Parameter(property = "cn1.backend.checkedCasts", defaultValue = "true") + private boolean checkedCasts; + + public void execute() throws MojoExecutionException, MojoFailureException { + File jdk8 = resolveJdk8(); + File work = new File(project.getBuild().getDirectory(), "cn1-backend"); + File classes = new File(work, "classes"); + File javaApi = new File(work, "javaapi-classes"); + File runtimeSources = new File(work, "runtime-src"); + File nativeSources = new File(work, "native"); + File translated = new File(work, "translated"); + // Emptied, not just created. Every one of these is derived, and nothing here + // removes a file that stopped being produced: a renamed or deleted source + // left its old .class behind, the translator still read it, and even + // requireMainClass accepted a main class the module no longer had -- so the + // package that came out was the previous implementation. Rebuilding from + // clean costs nothing, since neither the javac nor the clang pass below was + // ever incremental. + emptyDirs(classes, javaApi, runtimeSources, nativeSources, translated); + mkdirs(work, classes, javaApi, runtimeSources, nativeSources, translated); + + // The version of the runtime THIS MODULE compiles against, not the + // module's own: the sources handed to the translator have to be the same + // ones behind the classes the developer just built against, or the local + // run and the deployed binary are different programs. + String runtimeVersion = backendRuntimeVersion(); + File runtimeJar = resolve("com.codenameone", "codenameone-backend", + runtimeVersion, "parparvm-sources"); + unzip(runtimeJar, runtimeSources, nativeSources); + File parparvmBundle = resolve("com.codenameone", "codenameone-parparvm", + runtimeVersion, "bundle"); + File bundleDir = new File(work, "parparvm"); + mkdirs(bundleDir); + unzip(parparvmBundle, bundleDir, null); + File compilerJar = new File(bundleDir, "parparvm-compiler.jar"); + File javaApiJar = new File(bundleDir, "parparvm-java-api.jar"); + if (!compilerJar.isFile() || !javaApiJar.isFile()) { + throw new MojoExecutionException("The ParparVM bundle is missing its " + + "compiler or JavaAPI jar: " + parparvmBundle); + } + unzip(javaApiJar, javaApi, null); + + compile(jdk8, javaApi, runtimeSources, classes); + generateControllers(classes, work); + requireMainClass(classes); + translate(jdk8, compilerJar, javaApi, classes, nativeSources, translated); + File binary = output != null ? output + : new File(project.getBuild().getDirectory(), project.getArtifactId()); + link(translated, binary); + getLog().info("built " + binary); + } + + /** + * Compiles the module's sources and the runtime's against the JavaAPI as the + * BOOTCLASSPATH. See the class comment for why that matters. + */ + /** + * Generates the routers and the bootstrap for this module's `@RestController` + * classes, into the directory this goal has just compiled into. + * + * Not left to the `process-annotations` goal, which writes into Maven's + * target/classes: that is a different build, made against a JDK rather than + * against the backend's class library, and the translator never reads it. A + * router generated there would be absent from the binary while looking present + * in the project. Generating into the tree that is about to be translated is + * what makes the wiring real. + * + * Sets mainClass to the generated bootstrap when the module did not name one. + */ + private void generateControllers(File classes, File work) throws MojoExecutionException { + Map index; + try { + index = ClassScanner.scan(classes); + } catch (ProcessingException err) { + throw new MojoExecutionException("Could not scan the compiled backend classes: " + + err.getMessage(), err); + } + RestControllerAnnotationProcessor processor = new RestControllerAnnotationProcessor(); + ProcessorContext ctx = new ProcessorContext(classes, new File(work, "stubs"), index, + getLog(), project.getBasedir(), new Properties(), mainClass, + java.util.Collections.emptyList(), "UTF-8", + compileClasspathWithoutRuntime()); + try { + processor.start(ctx); + for (AnnotatedClass cls : index.values()) { + if (!cls.getClassAnnotations().isEmpty()) { + processor.processClass(cls, ctx); + } + } + processor.finish(ctx); + } catch (ProcessingException err) { + throw new MojoExecutionException("Could not process @RestController: " + + err.getMessage(), err); + } + if (ctx.hasErrors()) { + StringBuilder sb = new StringBuilder("@RestController could not be processed:"); + for (ProcessorContext.ProcessingError e : ctx.getErrors()) { + sb.append("\n ").append(e); + } + throw new MojoExecutionException(sb.toString()); + } + byte[] generated = ctx.getEmittedResources() + .get(RestControllerAnnotationProcessor.MAIN_CLASS_RESOURCE); + if (generated == null) { + return; + } + String name; + try { + name = new String(generated, "UTF-8").trim(); + } catch (java.io.UnsupportedEncodingException err) { + throw new MojoExecutionException("UTF-8 is required of every JDK", err); + } + if (mainClass == null || mainClass.length() == 0) { + mainClass = name; + getLog().info("cn1: entry point " + name + ", generated from @RestController"); + } + } + + /** + * Fails here, with the reason, rather than inside the translator. + * + * The compile below reads .java and only .java, on purpose: recompiling against + * the JavaAPI bootclasspath is what turns "this backend uses a class the runtime + * does not have" into a compile error instead of a link failure on the device, + * and reusing the jar Maven already built would give that up. The cost is that a + * main class written in Kotlin -- which `cn1:backend` runs happily, because that + * goal is a JVM launch -- never reaches this directory, and the translator's own + * complaint about it names neither Kotlin nor the reason. So say it plainly. The + * developer guide's "Limits worth knowing" carries the same statement. + */ + private void requireMainClass(File classes) throws MojoFailureException { + if (mainClass == null || mainClass.length() == 0) { + throw new MojoFailureException("No entry point: set , or annotate " + + "a class with @RestController and let the bootstrap be generated " + + "from it"); + } + if (new File(classes, mainClass.replace('.', '/') + ".class").isFile()) { + return; + } + throw new MojoFailureException("The main class " + mainClass + " was not " + + "produced by the backend compile. This goal compiles Java sources " + + "against the backend class library, so a main class written in " + + "Kotlin or generated into the build output is not visible to it " + + "yet -- write the entry point in Java, or keep it on the JVM with " + + "cn1:backend"); + } + + private void compile(File jdk8, File javaApi, File runtimeSources, File classes) + throws MojoExecutionException, MojoFailureException { + List sources = new ArrayList(); + for (Object root : project.getCompileSourceRoots()) { + collectJava(new File(String.valueOf(root)), sources); + } + collectJava(runtimeSources, sources); + if (sources.isEmpty()) { + throw new MojoFailureException("No Java sources to compile"); + } + + List command = new ArrayList(Arrays.asList( + new File(jdk8, "bin/javac").getAbsolutePath(), + "-nowarn", "-encoding", "UTF-8", + "-bootclasspath", javaApi.getAbsolutePath(), + "-source", "1.8", "-target", "1.8", + "-d", classes.getAbsolutePath())); + // The module's own dependencies, MINUS the backend runtime: its compiled + // form was built against a JDK, and the sources unpacked above are the + // half that belongs on this bootclasspath. + List classpath = new ArrayList(); + for (Object element : compileClasspathWithoutRuntime()) { + classpath.add(String.valueOf(element)); + } + if (!classpath.isEmpty()) { + command.add("-classpath"); + command.add(join(classpath, File.pathSeparator)); + } + command.addAll(sources); + run(command, project.getBasedir(), "compile the backend sources"); + stageResources(classes); + } + + /** + * Copies the module's resources in beside the classes just compiled. + * + * This directory is emptied and then filled from .java alone, and the project's + * own output directory is excluded from the translator input on purpose -- its + * classes were built against a JDK. The consequence was that anything read from + * the classpath, a properties or configuration file, was present under + * cn1:backend and simply absent from the packaged binary. Nothing failed at + * build time; the resource was just not there at runtime. + * + * Everything EXCEPT .class is taken, which is exactly the resources and none of + * the JDK-compiled code. + * + * ONLY Maven's processed output, never the raw resource directories. Those + * directories are what a / selects FROM, so copying them + * wholesale packaged the files the build was configured to leave out -- an + * environment file or a secret excluded on purpose would have gone into the + * executable, and the later overlay could not remove it. The processed copy is + * the answer Maven already computed. + */ + private void stageResources(File classes) throws MojoExecutionException { + File processed = new File(project.getBuild().getOutputDirectory()); + if (!processed.isDirectory()) { + // Nothing has processed the resources, so there are none to stage and + // nothing to guess at. Said out loud, because a resource silently absent + // from the binary is the failure this whole step exists to prevent. + if (!project.getBuild().getResources().isEmpty()) { + getLog().warn("cn1: this module declares resources but " + + processed + " does not exist, so none are packaged. Run " + + "process-resources first, or invoke this through the " + + "lifecycle rather than as a bare goal."); + } + return; + } + try { + int staged = copyNonClasses(processed, classes); + // Staged is not the same as READABLE, and the difference is silent. + // These files reach the translator, so anything that reads them at + // BUILD time works -- but the backend translates as app type "clean", + // and only the linux and windows types embed classpath resources into + // the binary. The clean runtime's Class.getResourceAsStream returns + // null unconditionally, so getResourceAsStream finds the file under + // cn1:backend, on the JVM, and finds nothing in the packaged + // executable. Said out loud rather than left to be discovered in + // production; embedding them is a change to the translator and the + // shared runtime, not to this goal. + if (staged > 0) { + getLog().warn("cn1: staged " + staged + " resource file(s) for translation, " + + "but a packaged backend cannot READ them: getResourceAsStream " + + "answers null in the translated runtime, though it works under " + + "cn1:backend. Read configuration from a file path or the " + + "environment instead of the classpath."); + } + } catch (IOException err) { + // A resource that cannot be staged is a packaging failure, not a note: + // the executable would be reported as built while missing something + // cn1:backend has, and the difference would first appear in production. + throw new MojoExecutionException("Could not stage the processed resources " + + "from " + processed + " into " + classes, err); + } + } + + /** @return how many non-class files were copied. */ + private int copyNonClasses(File from, File to) throws IOException { + if (from == null || !from.isDirectory()) { + return 0; + } + File[] children = from.listFiles(); + if (children == null) { + return 0; + } + int copied = 0; + for (File child : children) { + File target = new File(to, child.getName()); + if (child.isDirectory()) { + target.mkdirs(); + copied += copyNonClasses(child, target); + } else if (!child.getName().endsWith(".class")) { + copyFile(child, target); + copied++; + } + } + return copied; + } + + private static void copyFile(File from, File to) throws IOException { + InputStream in = new java.io.FileInputStream(from); + try { + OutputStream out = new java.io.FileOutputStream(to); + try { + byte[] chunk = new byte[8192]; + int n; + while ((n = in.read(chunk)) > 0) { + out.write(chunk, 0, n); + } + } finally { + out.close(); + } + } finally { + in.close(); + } + } + + private List compileClasspathWithoutRuntime() throws MojoExecutionException { + List out = new ArrayList(); + try { + for (Object element : project.getCompileClasspathElements()) { + String path = String.valueOf(element); + if (path.indexOf("codenameone-backend") >= 0) { + continue; + } + if (path.equals(project.getBuild().getOutputDirectory())) { + continue; + } + out.add(path); + } + } catch (Exception err) { + throw new MojoExecutionException("Could not resolve the compile classpath", err); + } + return out; + } + + private void translate(File jdk8, File compilerJar, File javaApi, File classes, + File nativeSources, File translated) + throws MojoExecutionException, MojoFailureException { + String simpleName = mainClass.substring(mainClass.lastIndexOf('.') + 1); + String packageName = mainClass.lastIndexOf('.') < 0 ? "" + : mainClass.substring(0, mainClass.lastIndexOf('.')); + + // The C has to be in the source root BEFORE the translator runs: it reads + // the directory to decide which native-only Java methods to keep, and the + // signature verifier checks every declared native against an actual + // implementation. + File sourceDir = new File(translated, "dist/" + simpleName + "-src"); + mkdirs(sourceDir); + copyDirectory(nativeSources, sourceDir); + + List command = new ArrayList(); + command.add(new File(jdk8, "bin/java").getAbsolutePath()); + if (sqlite) { + command.add("-Dcn1.sqlite=true"); + } + if (checkedCasts) { + command.add("-Dcn1.checkedCasts=true"); + } + command.add("-cp"); + command.add(compilerJar.getAbsolutePath()); + command.add("com.codename1.tools.translator.ByteCodeTranslator"); + command.add("clean"); + // The module's dependencies belong on the translator's input, not only on + // javac's classpath. Without them a backend that uses a type from another + // module -- the shared contract or DTO module the generated project + // recommends -- compiles here and then fails to translate, because javac + // resolved the type from a jar whose bytecode the translator never sees. + // The runtime is excluded for the same reason it is excluded from javac's + // classpath: its sources are compiled into `classes` already. + StringBuilder translatorInput = new StringBuilder(); + translatorInput.append(javaApi.getAbsolutePath()) + .append(';').append(classes.getAbsolutePath()); + for (String element : compileClasspathWithoutRuntime()) { + translatorInput.append(';').append(element); + } + command.add(translatorInput.toString()); + command.add(translated.getAbsolutePath()); + command.add(simpleName); + command.add(packageName); + command.add(simpleName); + command.add("1.0"); + command.add("clean"); + command.add("none"); + run(command, project.getBasedir(), "translate the backend to C"); + } + + private void link(File translated, File binary) + throws MojoExecutionException, MojoFailureException { + String simpleName = mainClass.substring(mainClass.lastIndexOf('.') + 1); + File sourceDir = new File(translated, "dist/" + simpleName + "-src"); + // Kept as a loud failure rather than dropped: the parameter names a real + // capability, and silently ignoring -Dcn1.backend.target would hand back a + // host binary labelled as a cross-compiled one. The script named here lives in + // the Codename One repository, not in a generated project, which is why the + // message says where it is instead of assuming it is on hand. + if (target != null && target.length() > 0) { + throw new MojoFailureException("cn1.backend.target is not supported from " + + "this goal yet: it builds for the machine it runs on. The " + + "cross-compiled targets (musl-x86_64, musl-arm64, glibc-x86_64, " + + "glibc-arm64) are produced by package.sh in the Codename One " + + "repository, which drives one container image per target; run " + + "this goal inside a container of the target flavour to get the " + + "same artifact here"); + } + List command = new ArrayList(Arrays.asList( + "clang", "-O3", "-w", + // Mandatory for generated C: Java arithmetic wraps, and clang -O3 + // provably miscompiles the output without these. + "-fwrapv", "-fno-strict-aliasing", + "-fno-builtin-fmod", "-fno-builtin-fmodf")); + if (!sqlite) { + // Turning the engine OFF is two changes, not one. Without + // -Dcn1.sqlite=true the translator leaves cn1_sqlite3.h out, but + // cn1_backend_db.c is copied and compiled either way -- and its + // SQLite branch includes that header unconditionally, so the compile + // fails with "cn1_sqlite3.h file not found" and the option advertised + // as saving the engine could not produce a binary at all. The macro + // is what compiles that file to stubs instead, which answer "could + // not open" and become an IOException, rather than dropping the Db + // natives and taking their Java methods with them. build.sh has + // always set both; this half had only the first. + command.add("-DCN1_BACKEND_NO_SQLITE"); + } + if (cflags != null && cflags.trim().length() > 0) { + command.addAll(Arrays.asList(cflags.trim().split("\\s+"))); + } + command.add("-I" + sourceDir.getAbsolutePath()); + File[] cFiles = sourceDir.listFiles(); + if (cFiles == null) { + throw new MojoExecutionException("The translator produced nothing in " + sourceDir); + } + for (File file : cFiles) { + String name = file.getName(); + // .S as well as .c, which is what vm/backend/build.sh compiles. The + // translator always emits cn1_virtual_thread_asm.S, and + // cn1_virtual_thread.c calls cn1VirtualThreadSwitch out of it, so a + // command that passed only .c reached the linker with that symbol + // undefined and this goal could not produce a binary at all. + // Generated resource assembly is in the same position. + if (name.endsWith(".c") || name.endsWith(".S") || name.endsWith(".s")) { + command.add(file.getAbsolutePath()); + } + } + command.addAll(Arrays.asList("-lm", "-lpthread", + "-lcurl", "-lssl", "-lcrypto", "-lnghttp2")); + command.add("-o"); + command.add(binary.getAbsolutePath()); + run(command, project.getBasedir(), "compile the generated C"); + } + + /** + * The codenameone-backend version this module depends on. + * + * Deliberately an error rather than a default when the dependency is absent: + * guessing a version here would translate a different runtime from the one the + * module was compiled and tested against. + */ + private String backendRuntimeVersion() throws MojoFailureException { + java.util.Set artifacts = project.getArtifacts(); + if (artifacts != null) { + for (Artifact artifact : artifacts) { + if ("com.codenameone".equals(artifact.getGroupId()) + && "codenameone-backend".equals(artifact.getArtifactId())) { + return artifact.getVersion(); + } + } + } + throw new MojoFailureException("This module does not depend on " + + "com.codenameone:codenameone-backend, so there is no backend " + + "runtime to translate. Add it as a dependency."); + } + + private File resolveJdk8() throws MojoFailureException { + if (jdk8Home != null && jdk8Home.length() > 0) { + File home = new File(jdk8Home); + if (new File(home, "bin/javac").isFile()) { + return home; + } + } + throw new MojoFailureException("A JDK 8 is required to translate; set " + + "JDK_8_HOME or -Dcn1.backend.jdk8"); + } + + private File resolve(String groupId, String artifactId, String version, String classifier) + throws MojoExecutionException { + Artifact artifact = repositorySystem.createArtifactWithClassifier( + groupId, artifactId, version, "jar", classifier); + ArtifactResolutionRequest request = new ArtifactResolutionRequest(); + request.setArtifact(artifact); + request.setLocalRepository(localRepository); + request.setRemoteRepositories(project.getRemoteArtifactRepositories()); + ArtifactResolutionResult result = repositorySystem.resolve(request); + if (!result.isSuccess() || artifact.getFile() == null) { + throw new MojoExecutionException("Could not resolve " + groupId + ":" + + artifactId + ":" + version + ":" + classifier); + } + return artifact.getFile(); + } + + /** + * Unpacks a jar. Entries under cn1-native/ go to `nativeTarget` when one is + * given, because the C belongs in the translator's source root rather than on + * the Java source path. + */ + private void unzip(File jar, File javaTarget, File nativeTarget) + throws MojoExecutionException { + try { + ZipFile zip = new ZipFile(jar); + try { + Enumeration entries = zip.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (entry.isDirectory()) { + continue; + } + String name = entry.getName(); + File destination; + if (name.startsWith("cn1-native/")) { + if (nativeTarget == null) { + continue; + } + destination = resolveInside(nativeTarget, + name.substring("cn1-native/".length()), jar, name); + } else if (name.startsWith("META-INF/")) { + continue; + } else { + destination = resolveInside(javaTarget, name, jar, name); + } + mkdirs(destination.getParentFile()); + InputStream in = zip.getInputStream(entry); + try { + copy(in, destination); + } finally { + in.close(); + } + } + } finally { + zip.close(); + } + } catch (IOException err) { + throw new MojoExecutionException("Could not unpack " + jar, err); + } + } + + /** + * The entry's destination, proven to be inside the directory it unpacks into. + * + * An archive entry name is attacker-controlled data, not a path this build + * chose: an entry called `../../../../etc/whatever` makes `new File(root, name)` + * resolve outside `root`, so unpacking writes wherever the entry says. That is + * Zip Slip, and here it would run with the developer's privileges during an + * ordinary `mvn package` against whatever jar the coordinates resolved to. + * + * Compared after canonicalisation rather than on the raw string, because `..` + * is not the only way out -- a symlinked parent resolves elsewhere too, and the + * textual check passes for both. The separator is appended to the root so a + * sibling whose name merely starts with it ("/tmp/outdir-evil" against + * "/tmp/outdir") cannot satisfy the prefix test. + */ + private static File resolveInside(File root, String relative, File jar, String entryName) + throws IOException { + File destination = new File(root, relative); + String prefix = root.getCanonicalPath() + File.separator; + String resolved = destination.getCanonicalPath(); + if (!resolved.startsWith(prefix)) { + throw new IOException("Refusing to unpack " + jar + ": entry \"" + entryName + + "\" resolves to " + resolved + ", outside " + root.getCanonicalPath()); + } + return destination; + } + + private static void copy(InputStream in, File destination) throws IOException { + OutputStream out = new FileOutputStream(destination); + try { + byte[] chunk = new byte[8192]; + int n; + while ((n = in.read(chunk)) > 0) { + out.write(chunk, 0, n); + } + } finally { + out.close(); + } + } + + private void copyDirectory(File from, File to) throws MojoExecutionException { + File[] children = from.listFiles(); + if (children == null) { + return; + } + for (File child : children) { + File destination = new File(to, child.getName()); + if (child.isDirectory()) { + mkdirs(destination); + copyDirectory(child, destination); + continue; + } + try { + InputStream in = new java.io.FileInputStream(child); + try { + copy(in, destination); + } finally { + in.close(); + } + } catch (IOException err) { + throw new MojoExecutionException("Could not copy " + child, err); + } + } + } + + private void collectJava(File dir, List out) { + File[] children = dir.listFiles(); + if (children == null) { + return; + } + for (File child : children) { + if (child.isDirectory()) { + collectJava(child, out); + } else if (child.getName().endsWith(".java")) { + out.add(child.getAbsolutePath()); + } + } + } + + private void run(List command, File directory, String what) + throws MojoExecutionException, MojoFailureException { + try { + ProcessBuilder builder = new ProcessBuilder(command); + builder.directory(directory); + builder.redirectErrorStream(true); + Process process = builder.start(); + StringBuilder output = new StringBuilder(); + InputStream in = process.getInputStream(); + byte[] chunk = new byte[8192]; + int n; + while ((n = in.read(chunk)) > 0) { + output.append(new String(chunk, 0, n, "UTF-8")); + } + int status = process.waitFor(); + if (status != 0) { + throw new MojoFailureException("Could not " + what + ":\n" + output); + } + if (output.length() > 0) { + getLog().debug(output.toString()); + } + } catch (IOException err) { + throw new MojoExecutionException("Could not " + what, err); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + throw new MojoExecutionException("Interrupted while trying to " + what, err); + } + } + + /** + * Removes each directory and its contents, and refuses to continue if one + * survives. + * + * The emptiness is the point, and it used to be assumed: File.delete returns + * false for a locked file on Windows or anything under a read-only directory, + * nothing looked at that, and the stale .class stayed where ClassScanner, + * requireMainClass and the translator would all find it. A controller or an + * entry point deleted from the source tree is then still packaged, so the + * build ships the previous implementation and says nothing. Checking the + * result rather than each delete catches every reason one can survive. + */ + private static void emptyDirs(File... dirs) throws MojoExecutionException { + for (File dir : dirs) { + deleteTree(dir); + if (dir == null || !dir.exists()) { + continue; + } + String[] left = dir.list(); + if (left != null && left.length > 0) { + throw new MojoExecutionException("Could not empty " + dir + + ": " + left.length + " entr" + (left.length == 1 ? "y" : "ies") + + " could not be deleted, and building over them would package " + + "classes that are no longer in the source tree."); + } + } + } + + private static void deleteTree(File file) { + if (file == null || !file.exists()) { + return; + } + File[] children = file.listFiles(); + if (children != null) { + for (File child : children) { + deleteTree(child); + } + } + // The result is checked by emptyDirs, which looks at what actually + // survived rather than at each delete: a directory that could not be + // removed but is empty is harmless, and one that still holds a class is + // not, whatever the reason. + file.delete(); + } + + private static void mkdirs(File... dirs) { + for (File dir : dirs) { + if (dir != null && !dir.isDirectory()) { + dir.mkdirs(); + } + } + } + + private static String join(List parts, String separator) { + StringBuilder out = new StringBuilder(); + for (int iter = 0; iter < parts.size(); iter++) { + if (iter > 0) { + out.append(separator); + } + out.append(parts.get(iter)); + } + return out.toString(); + } +} diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/BackendRunMojo.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/BackendRunMojo.java new file mode 100644 index 00000000000..3da203233ce --- /dev/null +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/BackendRunMojo.java @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugins.annotations.Execute; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.project.MavenProject; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Runs a backend module on this JVM: `mvn cn1:backend`. + * + * The point is speed. The same handler translated to a native binary takes about + * a minute and a half to build; here it starts in a couple of seconds, because + * the shared runtime (`codenameone-backend`) is ordinary Java compiled against + * the JDK and only the classes underneath it differ. Nothing in the protocol + * layer is a stand-in -- it is the same source that ships -- so what runs here + * behaves the way the deployed binary does. + * + * What this local runtime deliberately does NOT do is terminate TLS, and + * therefore serve HTTP/2: Tls and Http2 refuse with a message saying so. A second + * SSLEngine-based handshake would have its own bugs rather than production's, + * which is worse than not having it because it looks like coverage. Run + * `cn1:backend-package` when TLS is what you need to exercise. + */ +// Forks the lifecycle up to process-classes first, so `mvn cn1:backend` on its own +// does the obvious thing on a clean checkout instead of failing on an empty +// target/classes. +// +// process-classes rather than compile because that is where process-annotations is +// bound: a server written as @RestController classes has its router and its main +// GENERATED there, so stopping at compile would leave this goal looking for an entry +// point that the build had not produced yet. +@Execute(phase = LifecyclePhase.PROCESS_CLASSES) +@Mojo(name = "backend", requiresDependencyResolution = ResolutionScope.RUNTIME) +public class BackendRunMojo extends AbstractMojo { + + @Parameter(defaultValue = "${project}", readonly = true, required = true) + private MavenProject project; + + /** + * The class to run. Found automatically when the module has exactly one class + * with a main method, which is the usual shape. + */ + @Parameter(property = "cn1.backend.mainClass") + private String mainClass; + + /** Arguments for the program, space separated. */ + @Parameter(property = "cn1.backend.args") + private String args; + + /** Extra JVM options, space separated. */ + @Parameter(property = "cn1.backend.jvmArgs") + private String jvmArgs; + + public void execute() throws MojoExecutionException, MojoFailureException { + File classes = new File(project.getBuild().getOutputDirectory()); + if (!classes.isDirectory()) { + throw new MojoFailureException("Nothing is compiled in " + + classes + "; run `mvn compile` first, or `mvn compile cn1:backend`"); + } + + List classpath = new ArrayList(); + classpath.add(classes.getAbsolutePath()); + try { + for (Object element : project.getRuntimeClasspathElements()) { + String path = String.valueOf(element); + if (!classpath.contains(path)) { + classpath.add(path); + } + } + } catch (Exception err) { + throw new MojoExecutionException("Could not resolve the runtime classpath", err); + } + + String main = mainClass; + if (main == null || main.length() == 0) { + // The generator's own answer first. Annotation processing writes the + // entry point it created into META-INF/cn1-backend-main, and that is + // a statement of WHICH main to run -- scanning for main methods is a + // guess, and it fails the moment the module also holds a demo or a + // tool with one: the run is refused as ambiguous though the choice + // had already been made. cn1:backend-package reads the same marker, + // and the two must not disagree about what the module runs. + main = generatedMainClass(classes); + } + if (main == null || main.length() == 0) { + main = findMainClass(classes); + } + + List command = new ArrayList(); + command.add(javaExecutable()); + if (jvmArgs != null && jvmArgs.trim().length() > 0) { + command.addAll(Arrays.asList(jvmArgs.trim().split("\\s+"))); + } + command.add("-cp"); + command.add(join(classpath, File.pathSeparator)); + command.add(main); + if (args != null && args.trim().length() > 0) { + command.addAll(Arrays.asList(args.trim().split("\\s+"))); + } + + getLog().info("Running " + main + " on " + System.getProperty("java.version")); + try { + ProcessBuilder run = new ProcessBuilder(command); + run.directory(project.getBasedir()); + // Inherited rather than captured: a server logs as it serves, and a + // developer watching `cn1:backend` wants those lines as they happen. + // It also means Ctrl-C reaches the server, so its shutdown handler + // runs and in-flight requests finish. + run.inheritIO(); + Process process = run.start(); + int status = process.waitFor(); + if (status != 0) { + throw new MojoFailureException(main + " exited with status " + status); + } + } catch (IOException err) { + throw new MojoExecutionException("Could not start " + main, err); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + throw new MojoExecutionException("Interrupted while running " + main, err); + } + } + + /** + * The one class in this module with a main method. + * + * Deliberately an error when there are several rather than a guess: picking + * one and running it is how a developer ends up debugging the wrong process. + */ + /** + * The entry point annotation processing generated, or null when this module + * has none -- one written by hand, with no @RestController in it, has no + * marker and falls through to the scan below. + */ + private String generatedMainClass(File classesDir) { + File marker = new File(classesDir, + com.codename1.maven.processors.RestControllerAnnotationProcessor + .MAIN_CLASS_RESOURCE.replace('/', File.separatorChar)); + if (!marker.isFile()) { + return null; + } + try { + byte[] raw = new byte[(int) marker.length()]; + InputStream in = new java.io.FileInputStream(marker); + try { + int at = 0; + while (at < raw.length) { + int n = in.read(raw, at, raw.length - at); + if (n <= 0) { + break; + } + at += n; + } + } finally { + in.close(); + } + String name = new String(raw, "UTF-8").trim(); + return name.length() == 0 ? null : name; + } catch (IOException err) { + // Unreadable is not the same as absent, and the scan below still has + // a fair chance of being right; refusing outright would be worse. + getLog().warn("cn1: could not read " + marker + ": " + err); + return null; + } + } + + private String findMainClass(File classesDir) throws MojoFailureException { + List found = new ArrayList(); + collectMainClasses(classesDir, classesDir, found); + if (found.size() == 1) { + return found.get(0); + } + if (found.isEmpty()) { + throw new MojoFailureException("No class with a main method under " + + classesDir + "; set -Dcn1.backend.mainClass"); + } + throw new MojoFailureException("Several classes have a main method (" + + join(found, ", ") + "); choose one with -Dcn1.backend.mainClass"); + } + + private void collectMainClasses(File root, File dir, List found) { + File[] children = dir.listFiles(); + if (children == null) { + return; + } + for (File child : children) { + if (child.isDirectory()) { + collectMainClasses(root, child, found); + } else if (child.getName().endsWith(".class") && child.getName().indexOf('$') < 0) { + String name = child.getAbsolutePath() + .substring(root.getAbsolutePath().length() + 1) + .replace(File.separatorChar, '.'); + name = name.substring(0, name.length() - ".class".length()); + if (hasMainMethod(child)) { + found.add(name); + } + } + } + } + + /** + * Whether the class DECLARES `public static void main(String[])`. + * + * Read from the class file rather than by loading it: loading runs the static + * initialiser, and a backend's initialiser is as likely as not to open a + * socket or a database. The method table is read with ASM rather than by + * searching the bytes, because the constant pool of a class that merely CALLS + * main carries the same two strings. + */ + private boolean hasMainMethod(File classFile) { + final boolean[] found = new boolean[1]; + try { + InputStream in = new java.io.FileInputStream(classFile); + try { + new org.objectweb.asm.ClassReader(in).accept( + new org.objectweb.asm.ClassVisitor(org.objectweb.asm.Opcodes.ASM9) { + @Override + public org.objectweb.asm.MethodVisitor visitMethod(int access, + String name, String descriptor, String signature, + String[] exceptions) { + int wanted = org.objectweb.asm.Opcodes.ACC_PUBLIC + | org.objectweb.asm.Opcodes.ACC_STATIC; + if ("main".equals(name) + && "([Ljava/lang/String;)V".equals(descriptor) + && (access & wanted) == wanted) { + found[0] = true; + } + return null; + } + }, + org.objectweb.asm.ClassReader.SKIP_CODE + | org.objectweb.asm.ClassReader.SKIP_DEBUG + | org.objectweb.asm.ClassReader.SKIP_FRAMES); + } finally { + in.close(); + } + } catch (Exception err) { + return false; + } + return found[0]; + } + + private static String javaExecutable() { + File home = new File(System.getProperty("java.home")); + File candidate = new File(home, "bin/java"); + return candidate.isFile() ? candidate.getAbsolutePath() : "java"; + } + + private static String join(List parts, String separator) { + StringBuilder out = new StringBuilder(); + for (int iter = 0; iter < parts.size(); iter++) { + if (iter > 0) { + out.append(separator); + } + out.append(parts.get(iter)); + } + return out.toString(); + } +} diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestClientAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestClientAnnotationProcessor.java index 44ecf1a114a..af089f6b0a4 100644 --- a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestClientAnnotationProcessor.java +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestClientAnnotationProcessor.java @@ -611,7 +611,9 @@ static String extractResponsePayload(String paramSignature) { return jvmSignatureToJavaType(payload); } - private static String jvmSignatureToJavaType(String sig) { + /* package-private, not private: RestServerAnnotationProcessor generates the + server half of the same contract and needs the identical parsing. */ + static String jvmSignatureToJavaType(String sig) { if (sig == null || sig.length() == 0) return "java.lang.Object"; char c = sig.charAt(0); switch (c) { @@ -694,20 +696,26 @@ private static List splitTopLevelArgs(String args) { /// Strips top-level generic parameters from a Java type name so it can be /// used as a `Class` literal. `List` -> `List`. - private static String stripGeneric(String javaType) { + /* package-private, not private: RestServerAnnotationProcessor generates the + server half of the same contract and needs the identical parsing. */ + static String stripGeneric(String javaType) { if (javaType == null) return "java.lang.Object"; int lt = javaType.indexOf('<'); return lt < 0 ? javaType : javaType.substring(0, lt); } - private static boolean isCallbackType(String descriptor) { + /* package-private, not private: RestServerAnnotationProcessor generates the + server half of the same contract and needs the identical parsing. */ + static boolean isCallbackType(String descriptor) { return "Lcom/codename1/util/OnComplete;".equals(descriptor); } /// Returns the Java type name for a parameter, preferring the generic /// signature when available so `List` survives instead of erasing to /// `List`. - private static String javaTypeFor(Type asmType, String genericSig) { + /* package-private, not private: RestServerAnnotationProcessor generates the + server half of the same contract and needs the identical parsing. */ + static String javaTypeFor(Type asmType, String genericSig) { if (genericSig != null && genericSig.length() > 0) { return jvmSignatureToJavaType(genericSig); } @@ -732,17 +740,23 @@ private static String boxIfPrimitive(String type) { // Misc // ---------------------------------------------------------------- - private static String packageOf(String binary) { + /* package-private, not private: RestServerAnnotationProcessor generates the + server half of the same contract and needs the identical parsing. */ + static String packageOf(String binary) { int dot = binary.lastIndexOf('.'); return dot < 0 ? "" : binary.substring(0, dot); } - private static String simpleName(String binary) { + /* package-private, not private: RestServerAnnotationProcessor generates the + server half of the same contract and needs the identical parsing. */ + static String simpleName(String binary) { int dot = binary.lastIndexOf('.'); return dot < 0 ? binary : binary.substring(dot + 1); } - private static String escape(String s) { + /* package-private, not private: RestServerAnnotationProcessor generates the + server half of the same contract and needs the identical parsing. */ + static String escape(String s) { if (s == null) return ""; StringBuilder b = new StringBuilder(s.length() + 4); for (int i = 0; i < s.length(); i++) { @@ -753,7 +767,9 @@ private static String escape(String s) { return b.toString(); } - private static String sanitizeIdentifier(String s) { + /* package-private, not private: RestServerAnnotationProcessor generates the + server half of the same contract and needs the identical parsing. */ + static String sanitizeIdentifier(String s) { if (s == null || s.length() == 0) return "p"; StringBuilder b = new StringBuilder(s.length()); for (int i = 0; i < s.length(); i++) { diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java new file mode 100644 index 00000000000..b9f1a9caa2b --- /dev/null +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java @@ -0,0 +1,2111 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven.processors; + +import com.codename1.maven.annotations.AbstractAnnotationProcessor; +import com.codename1.maven.annotations.AnnotatedClass; +import com.codename1.maven.annotations.AnnotationValues; +import com.codename1.maven.annotations.JavaSourceCompiler; +import com.codename1.maven.annotations.MethodInfo; +import com.codename1.maven.annotations.ProcessingException; +import com.codename1.maven.annotations.ProcessorContext; +import com.codename1.maven.annotations.ClassScanner; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import org.objectweb.asm.Type; + +/** + * Generates a router for every `@RestController`, and the `main` that serves them. + * + * The shape is Spring's on purpose -- `@RestController`, `@GetMapping`, + * `@PathVariable`, `@RequestParam`, `@RequestBody`, `@ResponseStatus` mean here what + * they mean there -- so that reading one is enough to read the other. What differs is + * where the work happens: Spring resolves a route by walking a registry at request + * time, and this resolves it at build time into code that compares the request's own + * bytes. + * + * That is the reason to generate rather than reflect. A handwritten + * `if ("/healthz".equals(request.getTarget()))` builds a String for the target, + * hashes it and compares it, on every request and for every route it tests before the + * one that matches; and it is wrong the moment the client appends a query string. The + * generated form holds each route as a `byte[]` constant and asks the Request whether + * its path bytes are those bytes -- no String, no hash, and the query cannot break it. + * A route with no path variables therefore allocates nothing at all, which is what + * keeps the collector out of the request path. + * + * Reflection is not an option regardless: this code is translated to C, and + * `Class.forName` on an obfuscated name does not survive that. Generating source that + * the same compiler sees is what makes the wiring visible to the dead-code pass. + */ +public final class RestControllerAnnotationProcessor extends AbstractAnnotationProcessor { + + private static final String PKG = "Lcom/codename1/backend/annotations/"; + private static final String CONTROLLER = PKG + "RestController;"; + private static final String REQUEST_MAPPING = PKG + "RequestMapping;"; + private static final String PATH_VARIABLE = PKG + "PathVariable;"; + private static final String REQUEST_PARAM = PKG + "RequestParam;"; + private static final String REQUEST_HEADER = PKG + "RequestHeader;"; + private static final String REQUEST_BODY = PKG + "RequestBody;"; + private static final String RESPONSE_STATUS = PKG + "ResponseStatus;"; + + /** Mapping annotation to the HTTP method it stands for. */ + private static final Map MAPPINGS; + static { + Map m = new LinkedHashMap(); + m.put(PKG + "GetMapping;", "GET"); + m.put(PKG + "PostMapping;", "POST"); + m.put(PKG + "PutMapping;", "PUT"); + m.put(PKG + "DeleteMapping;", "DELETE"); + m.put(PKG + "PatchMapping;", "PATCH"); + MAPPINGS = Collections.unmodifiableMap(m); + } + + private static final String REQUEST_TYPE = "com.codename1.backend.HttpServer.Request"; + private static final String RESPONSE_TYPE = "com.codename1.backend.HttpServer.Response"; + + /** + * The same class as the DESCRIPTOR spells it. A nested class is + * Outer$Inner in bytecode, and the type derived from the descriptor keeps + * that -- so comparing against the dotted source spelling alone never + * matched, and both places that ask "is this a Response" were dead code: + * emitRoute never took the branch that SENDS one, and the encodable check + * never exempted it. Returning a Response from a controller, which the + * refusal message itself offers as the way to take control of the reply, + * did not work. + */ + private static final String RESPONSE_TYPE_BINARY = + "com.codename1.backend.HttpServer$Response"; + + /** Either spelling of HttpServer.Response. */ + private static boolean isResponseType(String javaType) { + return RESPONSE_TYPE.equals(javaType) || RESPONSE_TYPE_BINARY.equals(javaType); + } + + /** + * The verbs HttpServer routes. It compares them with equals and answers 501 + * to everything else before dispatch, so this list is the whole truth about + * what a generated route can be reached by. Kept in the same order the + * server declares it. + */ + /** + * The JDK types Json.writeValue has a branch for, and therefore the only ones + * a handler may return without writing itself. Kept in the order that method + * tests them so the two can be read side by side: String; the integral boxes; + * the floating ones; Map; List; byte[] (handled as an array before this); + * Collection, of which Set is the shape people actually return. + */ + private static final Set JSON_JDK_TYPES = Collections.unmodifiableSet( + new LinkedHashSet(Arrays.asList( + "java.lang.String", "java.lang.Character", + "java.lang.Boolean", "java.lang.Integer", "java.lang.Long", + "java.lang.Short", "java.lang.Byte", + "java.lang.Double", "java.lang.Float", + "java.util.Map", "java.util.HashMap", "java.util.LinkedHashMap", + "java.util.TreeMap", "java.util.SortedMap", + "java.util.List", "java.util.ArrayList", "java.util.LinkedList", + "java.util.Collection", "java.util.Set", "java.util.HashSet", + "java.util.LinkedHashSet", "java.util.TreeSet", "java.util.SortedSet"))); + + private static final List ROUTABLE_METHODS = Collections.unmodifiableList( + Arrays.asList("GET", "POST", "HEAD", "PUT", "DELETE", "PATCH", "OPTIONS")); + + /** Where the generated bootstrap's name is left for the packaging goal to read. */ + public static final String MAIN_CLASS_RESOURCE = "META-INF/cn1-backend-main"; + + private final TreeMap controllers = new TreeMap(); + + /** + * Every route shape seen so far, across every controller, to the method that + * claimed it. Kept beside `controllers` rather than inside one, because the + * generated bootstrap chains the routers and returns the first non-null + * response: two controllers colliding makes the later one unreachable in + * exactly the way two methods in one controller do. + */ + private final Map routeShapes = new LinkedHashMap(); + + /** Which controller claimed each shape, so a clash names the other one. */ + private final Map routeOwners = new LinkedHashMap(); + + private static final class Controller { + String binaryName; + String packageName; + String simpleName; + String routerSimpleName; + List basePaths = new ArrayList(); + List routes = new ArrayList(); + } + + private static final class Route { + String httpMethod; + String pattern; + String javaMethod; + String returnJavaType; + int status; + List params = new ArrayList(); + /** The literal bytes before the first `{`; the whole pattern when there is none. */ + String prefix; + /** For each variable, the literal that must follow it; "" when it runs to the end. */ + List after = new ArrayList(); + } + + private static final class Param { + String kind; // PATH, QUERY, HEADER, BODY, REQUEST + String name; + String javaType; + /** The same type with its arguments, when the method carried a signature. */ + String genericJavaType; + String defaultValue; + /** From the annotation. A request missing a required binding is refused. */ + boolean required; + /** Set when the body is decoded into a local before the call. */ + String local; + int variableIndex = -1; + } + + @Override + public Set getAnnotationDescriptors() { + return Collections.singleton(CONTROLLER); + } + + @Override + public void processClass(AnnotatedClass cls, ProcessorContext ctx) throws ProcessingException { + if (cls.getClassAnnotation(CONTROLLER) == null) { + return; + } + if (cls.isInterface() || cls.isAbstract()) { + ctx.error(cls, "@RestController must be a concrete class: " + cls.getBinaryName()); + return; + } + Controller controller = new Controller(); + controller.binaryName = cls.getBinaryName(); + controller.packageName = RestClientAnnotationProcessor.packageOf(controller.binaryName); + controller.simpleName = RestClientAnnotationProcessor.simpleName(controller.binaryName); + controller.routerSimpleName = controller.simpleName + "Router"; + controller.basePaths.addAll(pathsOf(cls.getClassAnnotation(REQUEST_MAPPING))); + if (controller.basePaths.isEmpty()) { + controller.basePaths.add(""); + } + if (!hasNoArgConstructor(cls)) { + ctx.error(cls, "@RestController needs a public no-argument constructor so the " + + "generated bootstrap can create it: " + controller.binaryName); + return; + } + + for (MethodInfo m : cls.getMethods()) { + if (m.isConstructor() || m.isSynthetic() || m.isStatic() || !m.isPublic()) { + continue; + } + String httpMethod = null; + List paths = null; + for (Map.Entry e : MAPPINGS.entrySet()) { + AnnotationValues values = m.getAnnotation(e.getKey()); + if (values != null) { + if (httpMethod != null) { + ctx.error(cls, "More than one mapping annotation on " + + controller.binaryName + "." + m.getName()); + return; + } + httpMethod = e.getValue(); + paths = pathsOf(values); + } + } + AnnotationValues mapping = m.getAnnotation(REQUEST_MAPPING); + if (mapping != null) { + if (httpMethod != null) { + ctx.error(cls, "@RequestMapping and a shorthand mapping on the same " + + "method: " + controller.binaryName + "." + m.getName()); + return; + } + httpMethod = mapping.getStringOrDefault("method", "GET"); + paths = pathsOf(mapping); + } + if (httpMethod == null) { + continue; + } + // The verb is emitted into the router verbatim, and HttpServer + // answers 501 to anything outside this set BEFORE dispatch -- so a + // mistyped or unsupported one compiles into a branch no request can + // ever reach, and both the build and the running server report + // success while the endpoint simply does not exist. Case matters + // for the same reason: the server compares with equals, so "get" + // is not "GET". + if (!ROUTABLE_METHODS.contains(httpMethod)) { + ctx.error(cls, controller.binaryName + "." + m.getName() + " maps HTTP " + + "method \"" + httpMethod + "\", which the server does not route: " + + "the request would be answered 501 before reaching it. Use one of " + + ROUTABLE_METHODS + ", in upper case."); + return; + } + if (paths.isEmpty()) { + paths = Collections.singletonList(""); + } + for (String base : controller.basePaths) { + for (String path : paths) { + Route route = buildRoute(cls, m, httpMethod, join(base, path), ctx); + if (route == null) { + return; + } + controller.routes.add(route); + } + } + } + if (controller.routes.isEmpty()) { + ctx.error(cls, "@RestController declares no mapped methods: " + controller.binaryName); + return; + } + if (!routeShapesAreDistinct(cls, controller, ctx)) { + return; + } + controllers.put(controller.binaryName, controller); + } + + /** + * Refuses two routes in one controller that no request can tell apart. + * + * A variable's NAME is not part of what the matcher sees, so `GET /notes/{id}` + * and `GET /notes/{name}` are one shape. The generated router tests the + * branches in order and returns from the first, which left the second method + * permanently unreachable with nothing at build time or run time saying so. + * + * The shapes are held across controllers, not just within one: the bootstrap + * chains the routers and takes the first non-null response, so a collision + * between two controllers is unreachable in precisely the same way. + */ + private boolean routeShapesAreDistinct(AnnotatedClass cls, Controller controller, + ProcessorContext ctx) { + for (int i = 0; i < controller.routes.size(); i++) { + Route route = controller.routes.get(i); + String shape = route.httpMethod + " " + route.pattern.replaceAll("\\{[^}]*\\}", "{}"); + String first = routeShapes.get(shape); + if (first != null) { + ctx.error(cls, controller.binaryName + "." + route.javaMethod + " and " + first + + " both answer " + shape + ". A path variable's NAME is not part of " + + "what a request carries, so nothing can tell them apart; the routers " + + "are tried in order and the second can never run. Give them different " + + "paths, or one method."); + return false; + } + // Within one controller a literal route is emitted before any variable + // route that would swallow it -- see the comparator in generateRouter. + // ACROSS controllers nothing orders them: the bootstrap tries the + // routers in turn and takes the first non-null, so a variable route in + // an alphabetically earlier controller answers a literal route's own + // path and that method never runs. There is no ordering to fix, since + // a router is a set of routes rather than a single pattern, so the + // ambiguity is reported instead of being resolved arbitrarily. + String clash = crossControllerClash(controller, route, shape); + if (clash != null) { + ctx.error(cls, clash); + return false; + } + routeShapes.put(shape, controller.binaryName + "." + route.javaMethod); + routeOwners.put(shape, controller.binaryName); + } + return true; + } + + /** + * Whether a route from another controller and this one can answer each other's + * paths, and the message saying so. + * + * Only ACROSS controllers: inside one, generateRouter's comparator already + * emits the literal route first. + */ + private String crossControllerClash(Controller controller, Route route, String shape) { + String mine = controller.binaryName; + for (Map.Entry e : routeOwners.entrySet()) { + // Same controller included. Skipping it assumed generateRouter's + // literal-first comparator settled everything inside one class, and it + // does not: two DYNAMIC patterns have no dominance, so "/a/{x}/c" and + // "/a/b/{y}" both answer /a/b/c and whichever the sort happens to emit + // first wins. That is the same ambiguity as across controllers, and it + // has the same answer. + String other = e.getKey(); + // Same verb, or they cannot collide at all -- except that a generated + // GET block also answers HEAD, so a GET route and a HEAD route on + // overlapping paths DO compete even though the verbs differ. + int mySpace = shape.indexOf(' '); + int otherSpace = other.indexOf(' '); + if (mySpace < 0 || otherSpace < 0) { + continue; + } + String myVerb = shape.substring(0, mySpace); + String otherVerb = other.substring(0, otherSpace); + boolean sameVerb = myVerb.equals(otherVerb); + boolean getAndHead = isGetHeadPair(myVerb, otherVerb); + if (!sameVerb && !getAndHead) { + continue; + } + if (!overlaps(other.substring(otherSpace + 1), shape.substring(mySpace + 1))) { + continue; + } + // Inside ONE controller, a wholly literal route and a dynamic one are + // resolved by generateRouter's comparator: it emits every route with + // no variables before every route with any, so /users/me is matched + // before /users/{id} and /users/42 still falls through to it. That + // pair is the single most ordinary thing to write, and it is what the + // message below tells people to do -- refusing it left no way to + // write it at all. + // Only that pair. Two DYNAMIC shapes have no dominance in that + // comparator, so "/a/{x}/c" against "/a/b/{y}" is still ambiguous, + // and two literals that overlap are the same literal twice. + // Within ONE controller a GET and a HEAD are ordered rather than + // ambiguous: generateRouter's comparator emits HEAD's own block ahead + // of GET's fallback, so the declared HEAD wins and the GET still + // answers everything else. Across controllers there is no such order + // -- the routers are tried in whatever sequence the bootstrap lists + // them -- so that pair is exactly as ambiguous as two GETs. + if (getAndHead && !sameVerb && mine.equals(e.getValue())) { + continue; + } + if (mine.equals(e.getValue()) && isLiteralShape(other) != isLiteralShape(shape)) { + continue; + } + return mine + "." + route.javaMethod + " answers " + shape + ", which " + + e.getValue() + " also answers as " + other + ". The routers are " + + "tried one after another, so whichever controller happens to " + + "come first takes the request and the other method never runs. " + + "Put both routes in one controller, where the more specific one " + + "is matched first, or give them different paths."; + } + return null; + } + + /** + * The class for an internal name as it appears on the COMPILE CLASSPATH, + * whether that is a directory of classes or a jar, or null when it is on + * neither. Read with ASM rather than loaded: a build must not run a + * dependency's static initialisers to answer a question about its shape. + */ + private static AnnotatedClass fromCompileClasspath(ProcessorContext ctx, String internalName) { + String entryName = internalName + ".class"; + for (String element : ctx.getCompileClasspath()) { + File file = new File(element); + if (file.isDirectory()) { + File candidate = new File(file, entryName.replace('/', File.separatorChar)); + if (candidate.isFile()) { + try { + return ClassScanner.readClass(candidate); + } catch (Exception err) { + return null; + } + } + continue; + } + if (!file.isFile()) { + continue; + } + try { + ZipFile zip = new ZipFile(file); + try { + ZipEntry entry = zip.getEntry(entryName); + if (entry != null) { + InputStream in = zip.getInputStream(entry); + try { + return ClassScanner.readClass(in, file); + } finally { + in.close(); + } + } + } finally { + zip.close(); + } + } catch (Exception err) { + continue; // an unreadable entry is not an answer + } + } + return null; + } + + /** + * Whether these two verbs are the GET/HEAD pair, in either order. + * + * They are not the same verb, but they answer the same requests: a generated + * GET block accepts HEAD, which is what makes a controller with only + * @GetMapping usable by a health check. + */ + private static boolean isGetHeadPair(String a, String b) { + return ("GET".equals(a) && "HEAD".equals(b)) || ("HEAD".equals(a) && "GET".equals(b)); + } + + /** The descriptor the scanner keys @Generated by. */ + private static final String GENERATED = PKG + "Generated;"; + + /** + * Whether a class of this name exists AND is somebody else's. + * + * The name being taken is not enough. An incremental build -- process-classes + * a second time, without a clean -- scans target/classes and finds the router + * this processor wrote on the FIRST pass, so an unconditional lookup reported + * the processor's own output as a class it would overwrite. Every project + * using @RestController failed its second build, and only a clean fixed it. + * Generated classes carry the marker for exactly this reason; a real + * user-defined collision has no marker and is still refused. + */ + private static boolean isNotOurOwnOutput(ProcessorContext ctx, String binaryName) { + AnnotatedClass existing = ctx.lookup(binaryName.replace('.', '/')); + return existing != null && !existing.getClassAnnotations().containsKey(GENERATED); + } + + /** HEAD sorts ahead of everything, so its own block precedes GET's fallback. */ + private static int methodRank(String httpMethod) { + return "HEAD".equals(httpMethod) ? 0 : 1; + } + + /** A shape with no variables at all, which the router matches before any. */ + private static boolean isLiteralShape(String shape) { + return shape.indexOf('{') < 0; + } + + /** + * Whether one path can satisfy both shapes. + * + * Two patterns that BOTH hold variables can still collide: "/a/{x}/c" and + * "/a/b/{y}" are different shapes, and "/a/b/c" is answered by either. An + * earlier version compared a variable pattern only against a literal one and + * returned early whenever both had a variable, which is exactly the case this + * misses. Segment by segment instead: a variable segment matches any single + * segment, so two shapes overlap when they have the same number of segments + * and every pair of segments is compatible. + */ + private static boolean overlaps(String left, String right) { + String[] a = left.split("/", -1); + String[] b = right.split("/", -1); + if (a.length != b.length) { + return false; + } + for (int i = 0; i < a.length; i++) { + if (!segmentsOverlap(a[i], b[i])) { + return false; + } + } + return true; + } + + /** + * Whether two single segments can be the same text. + * + * A segment is a literal, a whole variable, or a variable with literal text + * around it ("{}.json"). Two segments that both contain a variable overlap + * only when their fixed edges permit it; anything less certain errs toward + * reporting an ambiguity rather than shipping one. + */ + static boolean segmentsOverlap(String left, String right) { + boolean leftVar = left.indexOf("{}") >= 0; + boolean rightVar = right.indexOf("{}") >= 0; + if (!leftVar && !rightVar) { + return left.equals(right); + } + if (leftVar && rightVar) { + // The fixed EDGES decide it. Both carry a variable, but "{}.json" and + // "{}.xml" cannot both match one segment, and returning true here + // refused a controller that could never be ambiguous -- while the + // matcher this check guards handles literals around a variable + // perfectly well. A segment matching both must start with both + // prefixes and end with both suffixes, which is only possible when one + // of each pair contains the other. + String leftPrefix = left.substring(0, left.indexOf("{}")); + String rightPrefix = right.substring(0, right.indexOf("{}")); + String leftSuffix = left.substring(left.lastIndexOf("{}") + 2); + String rightSuffix = right.substring(right.lastIndexOf("{}") + 2); + boolean prefixesAgree = leftPrefix.startsWith(rightPrefix) + || rightPrefix.startsWith(leftPrefix); + boolean suffixesAgree = leftSuffix.endsWith(rightSuffix) + || rightSuffix.endsWith(leftSuffix); + return prefixesAgree && suffixesAgree; + } + String pattern = leftVar ? left : right; + String literal = leftVar ? right : left; + StringBuilder regex = new StringBuilder(); + for (int i = 0; i < pattern.length(); i++) { + if (pattern.startsWith("{}", i)) { + regex.append("[^/]+"); + i++; + } else { + char c = pattern.charAt(i); + if ("\\.[]{}()*+-?^$|".indexOf(c) >= 0) { + regex.append('\\'); + } + regex.append(c); + } + } + return literal.matches(regex.toString()); + } + + /** + * Splits a route pattern into the parts the generated matcher needs. + * + * `/notes/{id}/tags` becomes prefix `/notes/` and one variable followed by + * `/tags`. The prefix is what the byte compare rejects on, which is most + * requests for most routes, and it is why the split happens here rather than at + * request time. + */ + private Route buildRoute(AnnotatedClass cls, MethodInfo m, String httpMethod, String pattern, + ProcessorContext ctx) { + Route route = new Route(); + route.httpMethod = httpMethod; + route.pattern = pattern.length() == 0 ? "/" : pattern; + route.javaMethod = m.getName(); + + int firstVar = route.pattern.indexOf('{'); + route.prefix = firstVar < 0 ? route.pattern : route.pattern.substring(0, firstVar); + List variableNames = new ArrayList(); + int pos = firstVar; + while (pos >= 0) { + int close = route.pattern.indexOf('}', pos); + if (close < 0) { + ctx.error(cls, "Unclosed '{' in route " + route.pattern + " on " + + cls.getBinaryName() + "." + m.getName()); + return null; + } + variableNames.add(route.pattern.substring(pos + 1, close)); + int next = route.pattern.indexOf('{', close); + String following = next < 0 ? route.pattern.substring(close + 1) + : route.pattern.substring(close + 1, next); + // Two variables with nothing between them cannot be split. There is + // no text to look for, so the matcher gives the first one everything + // that is left and then fails because a second is still owed -- + // meaning the route compiles and then answers 404 to every request, + // which is the worst way to be wrong. Nothing can bind it, so it is + // refused where it is written. + if (following.length() == 0 && next >= 0) { + ctx.error(cls, cls.getBinaryName() + "." + m.getName() + " declares the " + + "route " + route.pattern + ", where two variables are adjacent. " + + "Nothing separates them, so no request could ever match it. Put a " + + "literal between them, such as a '/' or a '-'."); + return null; + } + route.after.add(following); + pos = next; + } + + Type[] paramTypes = Type.getArgumentTypes(m.getDescriptor()); + String[] genericParams = RestClientAnnotationProcessor.parseGenericParameterSignatures( + m.getSignature(), paramTypes.length); + List> paramAnnotations = m.getParameterAnnotations(); + for (int i = 0; i < paramTypes.length; i++) { + Param p = new Param(); + p.javaType = RestClientAnnotationProcessor.javaTypeFor(paramTypes[i], null); + // The ERASED name drives code generation below, because every check + // there compares against exact names like "java.util.Map". The + // generic form is kept separately, for validation only: the + // descriptor erases List to java.util.List, and accepting that + // is how a body of DTOs got through. + String genericType = genericParams == null || genericParams[i] == null + ? null + : RestClientAnnotationProcessor.javaTypeFor(paramTypes[i], genericParams[i]); + Map annotations = i < paramAnnotations.size() + ? paramAnnotations.get(i) : Collections.emptyMap(); + AnnotationValues pathVariable = annotations.get(PATH_VARIABLE); + AnnotationValues requestParam = annotations.get(REQUEST_PARAM); + AnnotationValues requestHeader = annotations.get(REQUEST_HEADER); + AnnotationValues requestBody = annotations.get(REQUEST_BODY); + if (pathVariable != null) { + p.kind = "PATH"; + p.name = pathVariable.getStringOrDefault("value", ""); + p.defaultValue = pathVariable.getStringOrDefault("defaultValue", ""); + p.variableIndex = variableNames.indexOf(p.name); + if (p.name.length() == 0) { + // Parameter names are not in the class file unless javac was told to + // keep them, and a router that guessed would bind the wrong value in + // silence. Naming it is one word and removes the whole question. + ctx.error(cls, "@PathVariable needs the variable name, as " + + "@PathVariable(\"id\"): " + cls.getBinaryName() + "." + + m.getName()); + return null; + } + if (p.variableIndex < 0) { + ctx.error(cls, "@PathVariable(\"" + p.name + "\") does not appear in the " + + "route " + route.pattern + " on " + cls.getBinaryName() + "." + + m.getName()); + return null; + } + } else if (requestParam != null) { + p.kind = "QUERY"; + p.name = requestParam.getStringOrDefault("value", ""); + p.defaultValue = requestParam.getStringOrDefault("defaultValue", ""); + p.required = requestParam.getBoolOrDefault("required", true); + if (p.name.length() == 0) { + ctx.error(cls, "@RequestParam needs the parameter name: " + + cls.getBinaryName() + "." + m.getName()); + return null; + } + } else if (requestHeader != null) { + p.kind = "HEADER"; + p.name = requestHeader.getStringOrDefault("value", ""); + p.defaultValue = requestHeader.getStringOrDefault("defaultValue", ""); + p.required = requestHeader.getBoolOrDefault("required", true); + if (p.name.length() == 0) { + ctx.error(cls, "@RequestHeader needs the header name: " + + cls.getBinaryName() + "." + m.getName()); + return null; + } + } else if (requestBody != null) { + p.kind = "BODY"; + p.required = requestBody.getBoolOrDefault("required", true); + } else if (REQUEST_TYPE.equals(p.javaType)) { + // The escape hatch: a handler that needs something this binding does not + // model takes the Request itself, exactly as it would have before. + p.kind = "REQUEST"; + } else { + ctx.error(cls, "Parameter " + (i + 1) + " of " + cls.getBinaryName() + "." + + m.getName() + " has no binding annotation. Annotate it with " + + "@PathVariable, @RequestParam, @RequestHeader or @RequestBody, " + + "or declare it as HttpServer.Request"); + return null; + } + p.genericJavaType = genericType; + String badKey = "BODY".equals(p.kind) ? unusableMapKey(genericType) : null; + if (badKey != null) { + // Separate from the element rule below, and with its own message, + // because Long is a perfectly good body VALUE -- every JSON + // integer arrives as one -- and only wrong as a KEY. The element + // check therefore approves Map, and the emitted + // shape check walks values() alone, so the map reached the + // handler with keys that violate its own declaration. + ctx.error(cls, "Cannot bind " + genericType + " from the body on " + + cls.getBinaryName() + "." + m.getName() + ". A JSON object's " + + "names are strings, so " + badKey + " keys arrive as String: " + + "iterating them as the declared type throws and get(" + badKey + + ") silently misses the value the client sent. Key the map by " + + "String."); + return null; + } + if ("BODY".equals(p.kind) && !bodyElementsAreDecoded(genericType)) { + ctx.error(cls, "Cannot bind " + genericType + " from the body on " + + cls.getBinaryName() + "." + m.getName() + ". A body is decoded " + + "by the JSON parser, which produces Map, List, String, Long, " + + "Double and Boolean -- so the elements arrive as Map and " + + "iterating them as the declared type throws, answering 500 " + + "from an endpoint that packaged cleanly. Take Map or " + + "List and convert, or use a @RestClient contract, which " + + "generates the codecs."); + return null; + } + if (!"REQUEST".equals(p.kind) && !isBindable(p.javaType, p.kind)) { + ctx.error(cls, "Cannot bind " + p.javaType + " from the request on " + + cls.getBinaryName() + "." + m.getName() + ". Path, query and " + + "header values bind to String and the primitive types; a body " + + "binds to String or java.util.Map"); + return null; + } + // A default that is not a value of the parameter's type. The generated + // to answers its fallback for anything unparseable, so + // defaultValue="oops" on an int became 0 -- and because the default is + // NON-EMPTY the required-value guard is skipped too, so an absent + // parameter called the handler with a number the controller never + // wrote. This is the author's own configuration, not a client's input, + // and it is wrong at build time or never. + if (p.defaultValue != null && p.defaultValue.length() > 0 + && !defaultParsesAs(p.javaType, p.defaultValue)) { + ctx.error(cls, cls.getBinaryName() + "." + m.getName() + " declares " + + "defaultValue=\"" + p.defaultValue + "\" for a " + p.javaType + + " parameter, which is not a " + p.javaType + ". It would be " + + "silently replaced by zero, and the handler would run on a " + + "value nobody wrote."); + return null; + } + route.params.add(p); + } + + // The generic signature, not just the descriptor: the descriptor erases + // List to java.util.List, and the check below would then approve + // the container without ever looking at what is IN it. + route.returnJavaType = RestClientAnnotationProcessor.javaTypeFor( + Type.getReturnType(m.getDescriptor()), returnSignature(m.getSignature())); + // A return type this router can actually turn into JSON. Anything else + // reached Json.write as an unknown object and came out as the QUOTED + // result of its toString() -- "com.example.Note@1a2b3c" where the caller + // expected an object -- while the build and the request both reported + // success. This processor has no DTO codec generation (the @RestClient + // half does), so the honest answer today is to refuse the shape rather + // than emit JSON nobody can use. + if (!isEncodableReturn(route.returnJavaType, ctx)) { + ctx.error(cls, cls.getBinaryName() + "." + m.getName() + " returns " + + route.returnJavaType + ", which the generated router cannot encode: " + + "it would be written as the JSON string of its toString(). Return a " + + "Map, a List, a Set, a String, a primitive, an HttpServer.Response, " + + "or make the type implement com.codename1.backend.Json.Writable."); + return null; + } + AnnotationValues status = m.getAnnotation(RESPONSE_STATUS); + // ResponseStatus documents that a value-returning method answers 200 and a + // void one answers 204. Defaulting to 200 for both made the annotation's + // own javadoc wrong about the case it exists to describe. + int implied = "void".equals(route.returnJavaType) ? 204 : 200; + route.status = status == null ? implied : status.getIntOrDefault("value", implied); + // A typo here is copied straight into the generated router, and neither + // writer questions it: HTTP/1 emits it as the status line and HTTP/2 + // submits it as :status, so a handler that worked perfectly answers with + // something the client rejects or cannot frame. Three digits is the whole + // of what HTTP defines. + if (route.status < 200 || route.status > 599) { + ctx.error(cls, cls.getBinaryName() + "." + m.getName() + " declares " + + "@ResponseStatus(" + route.status + "), which cannot be a handler's " + + "answer: a generated route sends ONE response, so it has to be a " + + "final status between 200 and 599. A 1xx is interim -- the client " + + "would go on waiting for the final response, and 101 is not legal " + + "over HTTP/2 at all."); + return null; + } + return route; + } + + /** + * Whether every type argument of a body type is something the JSON parser + * actually produces. It answers Map for an object, List for an array, and + * String/Long/Double/Boolean for the scalars -- so a List is a list of + * Map at runtime, and the first use of an element as a Note throws. + */ + /* package-private, not private: RestServerAnnotationProcessor decodes bodies + with the same parser and therefore needs the identical rule. One copy, so + the two halves cannot drift into disagreeing about what a body may hold. */ + static boolean bodyElementsAreDecoded(String javaType) { + if (javaType == null) { + return true; + } + int lt = javaType.indexOf('<'); + if (lt < 0) { + return true; // raw, so nothing was claimed + } + int end = javaType.lastIndexOf('>'); + if (end <= lt) { + return true; + } + List args = splitTypeArguments(javaType.substring(lt + 1, end)); + for (int i = 0; i < args.size(); i++) { + String arg = args.get(i); + if (arg.startsWith("?")) { + continue; // a wildcard claims nothing either + } + if (!PARSED_JSON_TYPES.contains(arg) && !bodyElementsAreDecoded(arg)) { + return false; + } + int inner = arg.indexOf('<'); + String rawArg = inner < 0 ? arg : arg.substring(0, inner); + if (!PARSED_JSON_TYPES.contains(rawArg)) { + return false; + } + } + return true; + } + + /** + * The first map key type in this declaration that a JSON body cannot produce, + * or null when every one of them is usable. + * + * Recursive, because the map need not be the outer type: List> has + * the same problem one level down. Object and a wildcard claim nothing, so + * they are fine; String is what actually arrives. + */ + static String unusableMapKey(String javaType) { + if (javaType == null) { + return null; + } + int lt = javaType.indexOf('<'); + int end = javaType.lastIndexOf('>'); + if (lt < 0 || end <= lt) { + return null; + } + List args = splitTypeArguments(javaType.substring(lt + 1, end)); + if ("java.util.Map".equals(javaType.substring(0, lt)) && args.size() == 2) { + // A BOUNDED wildcard is not the same as an unbounded one. Exempting + // anything beginning with '?' let Map through, + // and that bound is still a promise that every key is a Long -- so + // `for (Long key : body.keySet())` compiles and then fails on the + // Strings a JSON object really produces. Only the unbounded ? claims + // nothing; a bound is judged exactly like a spelled-out type. + String key = args.get(0).trim(); + if (!"?".equals(key)) { + // Already normalised by splitTypeArguments, so "? extends Long" + // arrives here as Long and only a genuinely unbounded wildcard + // is still spelled "?". One rule, in one place. + int inner = key.indexOf('<'); + String rawKey = inner < 0 ? key : key.substring(0, inner); + if (!"java.lang.String".equals(rawKey) + && !"java.lang.Object".equals(rawKey)) { + return rawKey; + } + } + } + for (int i = 0; i < args.size(); i++) { + String nested = unusableMapKey(args.get(i)); + if (nested != null) { + return nested; + } + } + return null; + } + + /** What Json.parse produces, and therefore all a body can be made of. */ + private static final Set PARSED_JSON_TYPES = Collections.unmodifiableSet( + new LinkedHashSet(Arrays.asList( + "java.lang.Object", "java.lang.String", "java.lang.Long", + "java.lang.Double", "java.lang.Boolean", + "java.util.Map", "java.util.List"))); + + private static boolean isBindable(String javaType, String kind) { + if ("BODY".equals(kind)) { + return "java.lang.String".equals(javaType) || "java.util.Map".equals(javaType) + || "java.util.List".equals(javaType); + } + return "java.lang.String".equals(javaType) || "int".equals(javaType) + || "long".equals(javaType) || "boolean".equals(javaType) + || "double".equals(javaType) || "float".equals(javaType) + || "short".equals(javaType) || "byte".equals(javaType); + } + + private static boolean hasNoArgConstructor(AnnotatedClass cls) { + for (MethodInfo m : cls.getMethods()) { + if (m.isConstructor() && m.isPublic() + && Type.getArgumentTypes(m.getDescriptor()).length == 0) { + return true; + } + } + return false; + } + + private static List pathsOf(AnnotationValues values) { + List out = new ArrayList(); + if (values == null) { + return out; + } + Object value = values.get("value"); + if (value instanceof List) { + for (Object item : (List) value) { + out.add(String.valueOf(item)); + } + } else if (value instanceof Object[]) { + for (Object item : (Object[]) value) { + out.add(String.valueOf(item)); + } + } else if (value != null) { + out.add(String.valueOf(value)); + } + return out; + } + + /** Joins a class-level base with a method-level path, without doubling the slash. */ + private static String join(String base, String path) { + String left = base == null ? "" : base.trim(); + String right = path == null ? "" : path.trim(); + while (left.endsWith("/")) { + left = left.substring(0, left.length() - 1); + } + // Every request target begins with "/", so a RELATIVE class prefix built + // a route no request could ever equal: @RequestMapping("api") plus + // "/users" produced "api/users", and /api/users answered 404 from an + // endpoint that packaged perfectly. The method-level half was normalised + // this way already; the class-level half was not. + if (left.length() > 0 && !left.startsWith("/")) { + left = "/" + left; + } + if (right.length() == 0) { + return left.length() == 0 ? "/" : left; + } + if (!right.startsWith("/")) { + right = "/" + right; + } + return left + right; + } + + @Override + public void finish(ProcessorContext ctx) throws ProcessingException { + if (ctx.hasErrors() || controllers.isEmpty()) { + return; + } + Map sources = new LinkedHashMap(); + for (Controller c : controllers.values()) { + String router = qualify(c.packageName, c.routerSimpleName); + // The same check the bootstrap gets below, for the same reason: a class + // of this name already in that package is overwritten in the output + // directory by the one compiled here, silently, because what is + // generated compiles perfectly well. Guarding only the bootstrap left + // every Router able to replace a real class. + if (isNotOurOwnOutput(ctx, router)) { + ctx.error(router + " already exists, and the router generated for " + + c.binaryName + " would replace it. Rename that class, or " + + "rename the controller."); + return; + } + sources.put(router, generateRouter(c)); + } + Controller first = controllers.values().iterator().next(); + String bootstrap = qualify(first.packageName, "BackendApplication"); + // A class of this name already in that package would be OVERWRITTEN in the + // output directory by the one compiled below -- silently, because the + // generated source compiles perfectly well. The packaged application then + // runs this bootstrap instead of the developer's own, dropping whatever + // startup it did: TLS, middleware, pooling. Refusing is the only safe + // answer, since there is no way to tell which one they meant. + if (isNotOurOwnOutput(ctx, bootstrap)) { + ctx.error(first.packageName + ".BackendApplication already " + + "exists, and the generated entry point would replace it. Rename " + + "that class, or move the controllers into another package."); + return; + } + sources.put(bootstrap, generateBootstrap(first.packageName)); + try { + List cp = new ArrayList(); + cp.add(ctx.getOutputClassDir()); + for (String element : ctx.getCompileClasspath()) { + cp.add(new File(element)); + } + JavaSourceCompiler.compile(sources, ctx.getOutputClassDir(), cp); + ctx.emitResource(MAIN_CLASS_RESOURCE, asciiBytes(bootstrap)); + } catch (IOException ioe) { + throw new ProcessingException("Could not compile the generated @RestController " + + "sources: " + ioe.getMessage(), ioe); + } + ctx.getLog().info("cn1: generated " + controllers.size() + " @RestController router(s) and " + + bootstrap); + } + + private static byte[] asciiBytes(String value) { + try { + return value.getBytes("UTF-8"); + } catch (UnsupportedEncodingException err) { + throw new IllegalStateException("UTF-8 is required of every VM", err); + } + } + + private static String qualify(String pkg, String simple) { + return pkg.length() == 0 ? simple : pkg + "." + simple; + } + + /** + * The router for one controller. + * + * Routes are emitted grouped by HTTP method and, within a group, static routes + * before dynamic ones. A static route is a single byte compare; a dynamic one + * pays for a prefix compare first, so an unrelated request leaves without + * allocating anything. + */ + private static String generateRouter(Controller c) { + StringBuilder sb = new StringBuilder(); + if (c.packageName.length() > 0) { + sb.append("package ").append(c.packageName).append(";\n\n"); + } + sb.append("// Generated from @RestController on ").append(c.binaryName) + .append(". Do not edit.\n"); + sb.append("@com.codename1.backend.annotations.Generated\n"); + sb.append("public final class ").append(c.routerSimpleName) + .append(" implements com.codename1.backend.HttpServer.Handler {\n\n"); + + List ordered = new ArrayList(c.routes); + Collections.sort(ordered, new java.util.Comparator() { + public int compare(Route a, Route b) { + // HEAD before GET, because a GET block also answers HEAD and + // dispatch returns on the first block that matches. Alphabetically + // GET comes first, which would have made a controller's explicit + // HEAD route unreachable the moment the GET fallback was added -- + // the fallback swallowing the specific case it defers to. + int byMethod = methodRank(a.httpMethod) - methodRank(b.httpMethod); + if (byMethod == 0) { + byMethod = a.httpMethod.compareTo(b.httpMethod); + } + if (byMethod != 0) { + return byMethod; + } + // Static routes first: they answer without touching the heap, and a + // dynamic route whose prefix also matches must not take the request + // from one that matches exactly. + int byKind = (a.after.isEmpty() ? 0 : 1) - (b.after.isEmpty() ? 0 : 1); + if (byKind != 0) { + return byKind; + } + return b.pattern.length() - a.pattern.length(); + } + }); + + for (int i = 0; i < ordered.size(); i++) { + Route route = ordered.get(i); + sb.append(" private static final byte[] P").append(i).append(" = ") + .append(byteArrayLiteral(route.after.isEmpty() ? route.pattern : route.prefix)) + .append("; // ").append(route.httpMethod).append(' ').append(route.pattern) + .append('\n'); + if (!route.after.isEmpty()) { + sb.append(" private static final String[] A").append(i).append(" = ") + .append(stringArrayLiteral(route.after)).append(";\n"); + } + } + + sb.append("\n private final ").append(c.simpleName).append(" impl;\n\n"); + sb.append(" public ").append(c.routerSimpleName).append("(").append(c.simpleName) + .append(" impl) {\n this.impl = impl;\n }\n\n"); + + // throws Exception because Handler does: a controller method that reads a + // database throws IOException, and a router that could not pass it on would + // force every handler to swallow its own errors. + sb.append(" public com.codename1.backend.HttpServer.Response handle(\n") + .append(" com.codename1.backend.HttpServer.Request request) throws Exception {\n"); + sb.append(" String httpMethod = request.getMethod();\n"); + // Checked ONCE, here, rather than inside the matcher: bindFrom answers a + // boolean, so a decoder that refused would only turn a malformed escape + // into "no route" -- a 404 for what is a syntax error the client can fix. + sb.append(" if (!wellFormedEscapes(request.getTarget())) {\n"); + sb.append(" return request.respond(400, \"text/plain; charset=utf-8\",\n"); + sb.append(" utf8(\"malformed percent-escape in the request target\"));\n"); + sb.append(" }\n"); + + String current = null; + boolean open = false; + for (int i = 0; i < ordered.size(); i++) { + Route route = ordered.get(i); + if (!route.httpMethod.equals(current)) { + if (open) { + sb.append(" }\n"); + } + sb.append(" if (\"").append(route.httpMethod) + .append("\".equals(httpMethod)"); + if ("GET".equals(route.httpMethod)) { + // A HEAD asks what a GET would answer, so a GET route is the + // route for it -- the server routes HEAD and its writer already + // suppresses the body and reports the length a GET would have + // sent. Without this, a controller declaring only @GetMapping + // answered 404 to every HEAD, which breaks the health checks and + // cache probes that use it, and disagrees with the Spring-style + // semantics these annotations borrow. An explicit @RequestMapping + // for HEAD still wins: its own block is emitted separately and + // dispatch returns on the first that matches. + sb.append(" || \"HEAD\".equals(httpMethod)"); + } + sb.append(") {\n"); + current = route.httpMethod; + open = true; + } + emitRoute(sb, route, i, c); + } + if (open) { + sb.append(" }\n"); + } + sb.append(" // No route here. Returning null lets the server answer 404, and\n"); + sb.append(" // lets another router be tried first when several are chained.\n"); + sb.append(" return null;\n }\n\n"); + emitRouterHelpers(sb); + sb.append("}\n"); + return sb.toString(); + } + + private static void emitRoute(StringBuilder sb, Route route, int index, Controller c) { + String pad = " "; + if (route.after.isEmpty()) { + sb.append(" if (request.pathIs(P").append(index).append(")) {\n"); + } else { + sb.append(" if (request.pathStartsWith(P").append(index).append(")) {\n"); + sb.append(pad).append("String[] bound = bindPath(request.pathFrom(P").append(index) + .append(".length), A").append(index).append(");\n"); + sb.append(pad).append("if (bound != null) {\n"); + pad = " "; + } + + emitRequiredGuards(sb, route, pad); + emitScalarGuards(sb, route, pad); + emitBodyLocals(sb, route, pad); + + StringBuilder args = new StringBuilder(); + for (int i = 0; i < route.params.size(); i++) { + Param p = route.params.get(i); + if (i > 0) { + args.append(", "); + } + args.append(argumentExpression(p)); + } + + String call = "impl." + route.javaMethod + "(" + args + ")"; + if ("void".equals(route.returnJavaType)) { + sb.append(pad).append(call).append(";\n"); + sb.append(pad).append("return request.respond(").append(route.status) + .append(", \"text/plain\", EMPTY);\n"); + } else if (isResponseType(route.returnJavaType)) { + // The handler built its own Response; a status annotation would be a lie + // about something this router no longer controls. + sb.append(pad).append("return ").append(call).append(";\n"); + } else if ("java.lang.String".equals(route.returnJavaType)) { + sb.append(pad).append("String result = ").append(call).append(";\n"); + sb.append(pad).append("return result == null ? request.respond(404, \"text/plain\", EMPTY)\n"); + sb.append(pad).append(" : request.respond(").append(route.status) + .append(", \"text/plain; charset=utf-8\", utf8(result));\n"); + } else { + // Everything else is JSON. respondJson writes into the connection's own + // Response, so a route that returns a value still allocates only that value. + sb.append(pad).append("Object result = ").append(call).append(";\n"); + sb.append(pad).append("return result == null ? request.respond(404, \"text/plain\", EMPTY)\n"); + sb.append(pad).append(" : request.respondJson(").append(route.status) + .append(", result);\n"); + } + + if (!route.after.isEmpty()) { + sb.append(" }\n"); + } + sb.append(" }\n"); + } + + /** + * Refuses a request that omits a binding declared required. + * + * Without this the `required` element of RequestParam, RequestHeader and + * RequestBody was read by nobody: an absent value simply converted to null, + * or to a primitive zero, and the handler ran as though the client had sent + * one. Both settings behaved identically, so the annotation documented a + * check that did not exist. A declared default supplies the value instead, + * so it makes the parameter satisfiable and no guard is emitted. + */ + private static void emitRequiredGuards(StringBuilder sb, Route route, String pad) { + for (int i = 0; i < route.params.size(); i++) { + Param p = route.params.get(i); + if (!p.required || (p.defaultValue != null && p.defaultValue.length() > 0)) { + continue; + } + String test; + String what; + if ("QUERY".equals(p.kind)) { + test = "request.queryParam(" + quote(p.name) + ") == null"; + what = "query parameter " + p.name; + } else if ("HEADER".equals(p.kind)) { + test = "request.getHeader(" + quote(p.name) + ") == null"; + what = "header " + p.name; + } else if ("BODY".equals(p.kind)) { + test = "request.getBody() == null || request.getBody().length() == 0"; + what = "request body"; + } else { + // A path variable cannot be absent: the route only matched because + // the segment was there. + continue; + } + sb.append(pad).append("if (").append(test).append(") {\n"); + sb.append(pad).append(" return request.respond(400, \"text/plain; charset=utf-8\",\n"); + sb.append(pad).append(" utf8(").append(quote("Missing required " + what)) + .append("));\n"); + sb.append(pad).append("}\n"); + } + } + + /** + * Decodes a structured body into a local, and refuses one that will not parse. + * + * bodyAsMap/bodyAsList answer null both for "there was no body" and for "the + * body was not JSON", and the call site could not tell those apart: malformed + * client JSON was handed to the controller as a null argument, so it surfaced + * as a 404, as a 500 from dereferencing it, or as a side effect performed with + * an argument the client never sent. Only the second case is a 400, so the + * emptiness test comes first and an absent body stays null for a binding that + * allows it. + */ + /** + * Refuses a scalar binding whose value is present but is not that type. + * + * "page=zz" for an int used to bind the annotation's defaultValue, so a client + * sending a typo got a different page rather than an error, and the handler + * could not tell the two apart. The annotation says defaultValue is "used when + * the request omits it", and a malformed value is not an omission -- the same + * shape of bug as an annotation element nothing reads. + * + * Numeric types only. toBoolean maps several spellings to true and everything + * else to false, which is a convention rather than a parse that can fail. + */ + private static void emitScalarGuards(StringBuilder sb, Route route, String pad) { + for (int i = 0; i < route.params.size(); i++) { + Param p = route.params.get(i); + String checker = numericChecker(p.javaType); + if (checker == null) { + continue; + } + String raw; + String what; + if ("PATH".equals(p.kind)) { + raw = "bound[" + p.variableIndex + "]"; + what = "path variable " + p.name; + } else if ("QUERY".equals(p.kind)) { + raw = "request.queryParam(" + quote(p.name) + ")"; + what = "query parameter " + p.name; + } else if ("HEADER".equals(p.kind)) { + raw = "request.getHeader(" + quote(p.name) + ")"; + what = "header " + p.name; + } else { + continue; + } + sb.append(pad).append("if (!").append(checker).append('(').append(raw) + .append(")) {\n"); + sb.append(pad).append(" return request.respond(400, \"text/plain; charset=utf-8\",\n"); + sb.append(pad).append(" utf8(").append(quote("The " + what + " is not a valid " + + p.javaType)).append("));\n"); + sb.append(pad).append("}\n"); + } + } + + /** Whether Json.write turns this return type into something other than toString(). */ + private static boolean isEncodableReturn(String javaType, ProcessorContext ctx) { + return isEncodableReturn(javaType, ctx, true); + } + + /** + * @param top whether this is the RETURN type itself rather than something + * inside it. void and HttpServer.Response are answers a route can + * give; they are not values Json can write. emitRoute handles a + * directly returned Response by sending it, so the exemption is + * real at the top and false anywhere else -- a + * List<Response> reaches the writer's fallback and each + * element is emitted as the quoted result of its toString(). + */ + private static boolean isEncodableReturn(String javaType, ProcessorContext ctx, + boolean top) { + if (javaType == null) { + return true; + } + if ("void".equals(javaType) || isResponseType(javaType)) { + return top; + } + // Arrays before anything else, because both tests below wave them + // through: a primitive array's name has no dot and a JDK array's name + // begins with "java.". Json writes byte[] as base64 and has no handling + // for any other array at all, so int[] or String[] reaches + // String.valueOf and is written as the JSON STRING "[I@1a2b3c" -- the + // array's identity, not its contents. + if (javaType.endsWith("[]")) { + return "byte[]".equals(javaType); + } + String raw = javaType; + int lt = raw.indexOf('<'); + if (lt >= 0) { + raw = raw.substring(0, lt); + // What Json actually writes is the ELEMENTS, so a container is only + // encodable when they are. java.util.List passes the raw check + // below on its own name, while every Note inside it comes out as + // the quoted result of its toString(). + int end = javaType.lastIndexOf('>'); + if (end > lt) { + List args = splitTypeArguments(javaType.substring(lt + 1, end)); + // A map's KEY is not written the way its values are. Json.writeValue + // calls String.valueOf on every key whatever its type, so + // Map comes back with keys spelled "[B@1a2b3c" and + // a Map with "com.example.Note@1a2b3c" -- object + // identity, not data, and different on every run. Checking the key + // as though it were a value approved both: byte[] and a writable + // DTO are perfectly good VALUES. This is the return-side twin of + // the rule that a JSON object's names arrive as strings. + if ("java.util.Map".equals(raw) && args.size() == 2) { + String key = args.get(0); + int keyLt = key.indexOf('<'); + String rawKey = keyLt < 0 ? key : key.substring(0, keyLt); + if (!"java.lang.String".equals(rawKey)) { + return false; + } + } + for (int i = 0; i < args.size(); i++) { + if (!isEncodableReturn(args.get(i), ctx, false)) { + return false; + } + } + } + } + // Only the JDK shapes Json ACTUALLY writes. "Anything under java." was too + // generous by a wide margin: java.util.Date reaches Json's final branch + // and comes back as a quoted, implementation-formatted toString(), and + // java.lang.Object holding a DTO comes back as "com.example.Note@1a2b3c" + // -- the same defect the DTO check exists to stop, arriving through a + // wider declared type. This list mirrors the branches of Json.writeValue + // in order; a type added there belongs here too. + if ("?".equals(raw)) { + // A wildcard is UNKNOWN, not primitive, and it has no dot -- so it fell + // into the branch below and was approved as though it were an int. The + // handler can then return a Date or a DTO inside a List and Json + // writes the quoted toString(), which is exactly what this validation + // refuses when the same thing is declared as List. Note the + // asymmetry with a BODY: an unknown element arriving is the client's + // to shape, while an unknown element leaving is ours to serialise. + return false; + } + if (raw.indexOf('.') < 0) { + return true; // a primitive, which is always written as one + } + if (JSON_JDK_TYPES.contains(raw)) { + return true; + } + if (raw.startsWith("java.")) { + return false; // some other JDK type Json would toString() + } + String internal = raw.replace('.', '/'); + AnnotatedClass cls = ctx.lookup(internal); + if (cls == null) { + // Not in the index because the index holds only what this project + // compiles -- so a DTO from a DEPENDENCY landed here and was waved + // through, and Json wrote it as the quoted result of its toString(). + // The compile classpath is where such a type actually lives, and it + // is read the same way the index was built: with ASM, so nothing is + // loaded and no static initialiser runs. + cls = fromCompileClasspath(ctx, internal); + } + if (cls == null) { + return false; // cannot be inspected, so cannot be trusted + } + for (String itf : cls.getInterfaceInternalNames()) { + if ("com/codename1/backend/Json$Writable".equals(itf)) { + return true; + } + } + return false; + } + + /** + * The return portion of a generic method signature, or null when the method + * carries none. Only the descriptor is guaranteed to exist, and it is the + * erased form. + */ + private static String returnSignature(String signature) { + if (signature == null) { + return null; + } + int close = signature.lastIndexOf(')'); + if (close < 0 || close + 1 >= signature.length()) { + return null; + } + return signature.substring(close + 1); + } + + /** + * Splits type arguments on their TOP-LEVEL commas, so the two arguments of + * Map<String, List<Note>> come back whole rather than being cut + * inside the nested one. + */ + /* package-private for the same reason as bodyElementsAreDecoded above. */ + static List splitTypeArguments(String args) { + List out = new ArrayList(); + int depth = 0; + int start = 0; + for (int i = 0; i < args.length(); i++) { + char c = args.charAt(i); + if (c == '<') { + depth++; + } else if (c == '>') { + depth--; + } else if (c == ',' && depth == 0) { + out.add(withoutWildcard(args.substring(start, i).trim())); + start = i + 1; + } + } + String last = args.substring(start).trim(); + if (last.length() > 0) { + out.add(withoutWildcard(last)); + } + return out; + } + + /** + * A type argument reduced to what it actually PROMISES about the value. + * + * Normalised here, at the one place type arguments are produced, rather than + * at each of the six consumers -- validation, the element type, the map value + * type, the encodability walk and the emitted instanceof checks all ask the + * same question, and a bounded wildcard was being read as "claims nothing" by + * every one of them. `List` was accepted with no runtime + * check at all, so `[1]` reached the handler as a list holding a Long and the + * first typed read answered 500 where a 400 was owed. + * + * `? extends T` promises T. `? super T` does NOT: the value may be T or any + * supertype of it, so the only honest reading is the unbounded one, and a + * check against T there would reject values the declaration allows. + */ + private static String withoutWildcard(String arg) { + if (arg.startsWith("? extends ")) { + return arg.substring("? extends ".length()).trim(); + } + if (arg.startsWith("? super ")) { + return "?"; + } + return arg; + } + + /** Whether an annotation's declared default really is a value of that type. */ + private static boolean defaultParsesAs(String javaType, String value) { + String v = value.trim(); + try { + if ("int".equals(javaType)) { + Integer.parseInt(v); + } else if ("long".equals(javaType)) { + Long.parseLong(v); + } else if ("short".equals(javaType)) { + Short.parseShort(v); + } else if ("byte".equals(javaType)) { + Byte.parseByte(v); + } else if ("double".equals(javaType)) { + // parseDouble answers infinity for 1e999 rather than throwing, so + // this branch used to approve a default the RUNTIME guard rejects + // in the identical spelling: omit the value and the generated + // converter hands the controller an infinity, send it and the + // request is a 400. The same rule as the request path, then, and + // the same test -- the SPELLING decides whether an infinity was + // meant, because the parsed value cannot tell 1e999 from Infinity. + double d = Double.parseDouble(v); + return !Double.isInfinite(d) || spellsInfinity(v); + } else if ("float".equals(javaType)) { + // Same rule, and it was wrong here in the other direction: + // Double.isInfinite was the "did they mean it" test, and + // Double.parseDouble("1e999") is itself infinite, so every + // double-overflowing default was read as a deliberate infinity. + double d = Double.parseDouble(v); + return !Float.isInfinite((float) d) || spellsInfinity(v); + } else if ("boolean".equals(javaType)) { + // The binder accepts only these two, so a default of "yes" would + // bind false and read as a deliberate choice. + return "true".equalsIgnoreCase(v) || "false".equalsIgnoreCase(v); + } + return true; // String and anything else: no parsing + } catch (NumberFormatException err) { + return false; + } + } + + /** + * Whether this text asks for an infinity, rather than merely producing one. + * + * The same test the generated guards use, deliberately: parseDouble answers + * infinity for both "Infinity" and "1e999", so only the text tells the two + * apart, and a default and a request value that disagreed about which is + * which is exactly the divergence this pair of rules exists to prevent. + */ + private static boolean spellsInfinity(String value) { + return value.trim().indexOf("Infinity") >= 0; + } + + private static String numericChecker(String javaType) { + if ("boolean".equals(javaType)) return "parsesBoolean"; + if ("int".equals(javaType)) return "parsesInt"; + if ("long".equals(javaType)) return "parsesLong"; + if ("double".equals(javaType)) return "parsesDouble"; + if ("float".equals(javaType)) return "parsesFloat"; + if ("short".equals(javaType)) return "parsesShort"; + if ("byte".equals(javaType)) return "parsesByte"; + return null; + } + + private static void emitBodyLocals(StringBuilder sb, Route route, String pad) { + for (int i = 0; i < route.params.size(); i++) { + Param p = route.params.get(i); + if (!"BODY".equals(p.kind) || "java.lang.String".equals(p.javaType)) { + continue; + } + boolean map = "java.util.Map".equals(p.javaType); + String type = map ? "java.util.Map" : "java.util.List"; + String decoder = map ? "bodyAsMap" : "bodyAsList"; + p.local = "body" + i; + sb.append(pad).append(type).append(' ').append(p.local).append(" = null;\n"); + sb.append(pad).append("if (request.getBody() != null && request.getBody().length() > 0) {\n"); + sb.append(pad).append(" ").append(p.local).append(" = ").append(decoder) + .append("(request.getBody());\n"); + sb.append(pad).append(" if (").append(p.local).append(" == null) {\n"); + sb.append(pad).append(" return request.respond(400, \"text/plain; charset=utf-8\",\n"); + sb.append(pad).append(" utf8(\"The request body is not valid JSON\"));\n"); + sb.append(pad).append(" }\n"); + // Declaring List does not make the ELEMENTS strings. The + // build-time check says the declared element type is one the parser + // can produce; what it actually produced depends on what the client + // sent, so "[1]" fills a List with a Long and the handler's + // first read of it throws -- turning a malformed request into a 500 + // instead of the 400 it is. Checked with instanceof, never a cast: + // a failed cast does not throw in the packaged runtime at all. + // Maps as well as lists. A Map that receives + // {"value":1} holds a Long under a String declaration, and the + // handler's first typed read throws -- the same 500-for-a-400 the + // list case had, skipped only because the check was written for + // lists and the map branch went past it. + emitShapeChecks(sb, pad + " ", p.local, p.genericJavaType, 0); + sb.append(pad).append("}\n"); + } + } + + /** + * Emits the runtime element checks for one declared container, and for + * whatever its elements are declared to contain, to whatever depth the + * declaration goes. Each level is a loop; the innermost is an instanceof. + * + * instanceof rather than a cast at every level, because a failed cast does + * not throw in the packaged runtime -- the wrong object is simply handed on. + */ + private static void emitShapeChecks(StringBuilder sb, String pad, String expr, + String genericJavaType, int depth) { + // No depth cutoff. One used to stop emitting below the fifth level while + // build-time validation accepted the whole shape, so a declaration nested + // deeper than that was checked partway and the rest reached the handler + // unverified -- a 500 for what is a 400, at exactly the depth nobody + // looks. The declaration is finite, so the recursion is too. + if (genericJavaType != null && genericJavaType.startsWith("java.util.Map<")) { + String value = mapBodyValueType(genericJavaType); + if (value != null) { + String raw = value.indexOf('<') < 0 ? value + : value.substring(0, value.indexOf('<')); + String var = "v" + depth + "$"; + sb.append(pad).append("for (java.util.Iterator it").append(depth) + .append("$ = ").append(expr).append(".values().iterator(); it") + .append(depth).append("$.hasNext();) {\n"); + sb.append(pad).append(" Object ").append(var).append(" = it") + .append(depth).append("$.next();\n"); + sb.append(pad).append(" if (").append(var).append(" != null && !(") + .append(var).append(" instanceof ").append(raw).append(")) {\n"); + sb.append(pad).append(" return request.respond(400, " + + "\"text/plain; charset=utf-8\",\n"); + sb.append(pad).append(" utf8(") + .append(quote("A value of the request body is not a " + raw)) + .append("));\n"); + sb.append(pad).append(" }\n"); + if (value.indexOf('<') >= 0 && depth < 32) { + sb.append(pad).append(" if (").append(var).append(" != null) {\n"); + emitShapeChecks(sb, pad + " ", "((" + raw + ")" + var + ")", + value, depth + 1); + sb.append(pad).append(" }\n"); + } + sb.append(pad).append("}\n"); + } + return; + } + emitElementChecks(sb, pad, expr, genericJavaType, depth); + } + + /** A map body's declared value type, when it is one worth asserting. */ + private static String mapBodyValueType(String genericJavaType) { + int lt = genericJavaType.indexOf('<'); + int end = genericJavaType.lastIndexOf('>'); + if (lt < 0 || end <= lt) { + return null; + } + List args = splitTypeArguments(genericJavaType.substring(lt + 1, end)); + if (args.size() != 2) { + return null; + } + String value = args.get(1); + String raw = value.indexOf('<') < 0 ? value : value.substring(0, value.indexOf('<')); + return PARSED_JSON_TYPES.contains(raw) && !"java.lang.Object".equals(raw) + ? value : null; + } + + private static void emitElementChecks(StringBuilder sb, String pad, String expr, + String genericJavaType, int depth) { + String element = bodyElementType(genericJavaType); + if (element == null) { + return; // nothing declared to check + } + String var = "e" + depth + "$"; + String index = "i" + depth + "$"; + sb.append(pad).append("for (int ").append(index).append(" = 0; ").append(index) + .append(" < ").append(expr).append(".size(); ").append(index).append("++) {\n"); + sb.append(pad).append(" Object ").append(var).append(" = ").append(expr) + .append(".get(").append(index).append(");\n"); + String raw = element.indexOf('<') < 0 ? element + : element.substring(0, element.indexOf('<')); + sb.append(pad).append(" if (").append(var).append(" != null && !(").append(var) + .append(" instanceof ").append(raw).append(")) {\n"); + sb.append(pad).append(" return request.respond(400, " + + "\"text/plain; charset=utf-8\",\n"); + sb.append(pad).append(" utf8(") + .append(quote("An element of the request body is not a " + raw)).append("));\n"); + sb.append(pad).append(" }\n"); + if (element.indexOf('<') >= 0 && depth < 32) { + sb.append(pad).append(" if (").append(var).append(" != null) {\n"); + emitShapeChecks(sb, pad + " ", "((" + raw + ")" + var + ")", + element, depth + 1); + sb.append(pad).append(" }\n"); + } + sb.append(pad).append("}\n"); + } + + /** + * The element type of a declared List or Set body, when it is one the runtime + * check can assert -- a type the parser produces, or another container whose + * own elements can then be checked. Null for a raw container, a wildcard, or + * anything else, where there is nothing to assert. + */ + private static String bodyElementType(String genericJavaType) { + if (genericJavaType == null) { + return null; + } + int lt = genericJavaType.indexOf('<'); + int end = genericJavaType.lastIndexOf('>'); + if (lt < 0 || end <= lt) { + return null; + } + String raw = genericJavaType.substring(0, lt); + if (!"java.util.List".equals(raw) && !"java.util.Set".equals(raw) + && !"java.util.Collection".equals(raw)) { + return null; + } + List args = splitTypeArguments(genericJavaType.substring(lt + 1, end)); + if (args.size() != 1) { + return null; + } + String arg = args.get(0); + // A nested container counts: its own elements are checked one level in. + String argRaw = arg.indexOf('<') < 0 ? arg : arg.substring(0, arg.indexOf('<')); + return PARSED_JSON_TYPES.contains(argRaw) && !"java.lang.Object".equals(argRaw) + ? arg : null; + } + + private static String argumentExpression(Param p) { + if ("REQUEST".equals(p.kind)) { + return "request"; + } + String raw; + if ("PATH".equals(p.kind)) { + raw = "bound[" + p.variableIndex + "]"; + } else if ("QUERY".equals(p.kind)) { + raw = "request.queryParam(" + quote(p.name) + ")"; + } else if ("HEADER".equals(p.kind)) { + raw = "request.getHeader(" + quote(p.name) + ")"; + } else { + return bodyExpression(p); + } + return convert(p.javaType, raw, p.defaultValue); + } + + private static String bodyExpression(Param p) { + if ("java.lang.String".equals(p.javaType)) { + return "request.getBody()"; + } + if (p.local != null) { + return p.local; + } + if ("java.util.Map".equals(p.javaType)) { + return "bodyAsMap(request.getBody())"; + } + return "bodyAsList(request.getBody())"; + } + + /** + * Converts a raw request value to the parameter's type. + * + * An absent value takes the annotation's default rather than throwing, which is + * what Spring does and what a caller expects from `defaultValue`. + */ + private static String convert(String javaType, String raw, String defaultValue) { + String fallback = defaultValue == null ? "" : defaultValue; + if ("java.lang.String".equals(javaType)) { + return fallback.length() == 0 ? raw + : "orDefault(" + raw + ", " + quote(fallback) + ")"; + } + String zero = "boolean".equals(javaType) ? "false" : "0"; + String literalDefault = fallback.length() == 0 ? zero + : "to" + capitalize(javaType) + "(" + quote(fallback) + ", " + zero + ")"; + return "to" + capitalize(javaType) + "(" + raw + ", " + literalDefault + ")"; + } + + private static String capitalize(String javaType) { + return Character.toUpperCase(javaType.charAt(0)) + javaType.substring(1); + } + + /** + * The helpers every router shares. Emitted into each router rather than into a + * runtime class so that a module with no controllers links none of it, and so + * the dead-code pass can drop whichever ones this controller never calls. + */ + private static void emitRouterHelpers(StringBuilder sb) { + sb.append(" private static final byte[] EMPTY = new byte[0];\n\n"); + sb.append(" /**\n"); + sb.append(" * Binds the path variables, or returns null when the rest of the path is\n"); + sb.append(" * not this route after all. `after[i]` is the literal that follows\n"); + sb.append(" * variable i, empty when the variable runs to the end.\n"); + sb.append(" */\n"); + sb.append(" private static String[] bindPath(String rest, String[] after) {\n"); + sb.append(" String[] out = new String[after.length];\n"); + sb.append(" return bindFrom(rest, after, 0, 0, out) ? out : null;\n"); + sb.append(" }\n\n"); + + sb.append(" /**\n"); + sb.append(" * Tries every placement of the remaining literals, not just the first.\n"); + sb.append(" *\n"); + sb.append(" * A variable's value may contain the literal that follows it: matching\n"); + sb.append(" * /download/{name}.json against \"foo.json.json\" has to bind name to\n"); + sb.append(" * \"foo.json\", and taking the first occurrence bound it to \"foo\", left\n"); + sb.append(" * \".json\" unconsumed and rejected a request the route does match.\n"); + sb.append(" */\n"); + sb.append(" private static boolean bindFrom(String rest, String[] after, int i, int pos,\n"); + sb.append(" String[] out) {\n"); + sb.append(" if (i == after.length) {\n"); + sb.append(" return pos == rest.length();\n"); + sb.append(" }\n"); + sb.append(" String literal = after[i];\n"); + sb.append(" if (literal.length() == 0) {\n"); + sb.append(" String value = rest.substring(pos);\n"); + sb.append(" if (value.length() == 0 || value.indexOf('/') >= 0) {\n"); + sb.append(" return false;\n"); + sb.append(" }\n"); + sb.append(" out[i] = decode(value);\n"); + sb.append(" return i + 1 == after.length;\n"); + sb.append(" }\n"); + sb.append(" for (int at = rest.indexOf(literal, pos) ; at >= 0 ;\n"); + sb.append(" at = rest.indexOf(literal, at + 1)) {\n"); + sb.append(" String value = rest.substring(pos, at);\n"); + sb.append(" if (value.length() == 0) {\n"); + sb.append(" continue;\n"); + sb.append(" }\n"); + sb.append(" // A variable is one segment. Without this, /notes/{id} would\n"); + sb.append(" // match /notes/1/2 and hand the method \"1/2\" as the id. Every\n"); + sb.append(" // later occurrence spans this slash too, so stop rather than\n"); + sb.append(" // continue.\n"); + sb.append(" if (value.indexOf('/') >= 0) {\n"); + sb.append(" break;\n"); + sb.append(" }\n"); + sb.append(" out[i] = decode(value);\n"); + sb.append(" if (bindFrom(rest, after, i + 1, at + literal.length(), out)) {\n"); + sb.append(" return true;\n"); + sb.append(" }\n"); + sb.append(" }\n"); + sb.append(" return false;\n"); + sb.append(" }\n\n"); + + sb.append(" /**\n"); + sb.append(" * Percent-decodes one path segment. The octets are gathered and decoded\n"); + sb.append(" * as a run: an escape carries one byte of UTF-8, and decoding them one\n"); + sb.append(" * at a time turns every non-ASCII value into mojibake.\n"); + sb.append(" */\n"); + sb.append(" private static String decode(String value) {\n"); + sb.append(" if (value.indexOf('%') < 0) {\n"); + sb.append(" return value;\n"); + sb.append(" }\n"); + sb.append(" byte[] out = new byte[value.length()];\n"); + sb.append(" int length = 0;\n"); + sb.append(" for (int i = 0 ; i < value.length() ; i++) {\n"); + sb.append(" char c = value.charAt(i);\n"); + sb.append(" if (c == '%' && i + 2 < value.length()) {\n"); + sb.append(" int hi = hex(value.charAt(i + 1));\n"); + sb.append(" int lo = hex(value.charAt(i + 2));\n"); + sb.append(" if (hi >= 0 && lo >= 0) {\n"); + sb.append(" out[length++] = (byte)((hi << 4) | lo);\n"); + sb.append(" i += 2;\n"); + sb.append(" continue;\n"); + sb.append(" }\n"); + sb.append(" }\n"); + sb.append(" out[length++] = (byte)c;\n"); + sb.append(" }\n"); + sb.append(" try {\n"); + sb.append(" return new String(out, 0, length, \"UTF-8\");\n"); + sb.append(" } catch (java.io.UnsupportedEncodingException err) {\n"); + sb.append(" return new String(out, 0, length);\n"); + sb.append(" }\n"); + sb.append(" }\n\n"); + sb.append(" /**\n"); + sb.append(" * Every % in a target must introduce two hex digits.\n"); + sb.append(" *\n"); + sb.append(" * A malformed escape used to decode as literal text, so /users/%ZZ\n"); + sb.append(" * reached the handler as those four characters -- and /users/%252F,\n"); + sb.append(" * a correctly escaped %2F, arrived as whatever a raw %2F would.\n"); + sb.append(" * Aliasing like that is how a check in front of a handler is passed\n"); + sb.append(" * by one spelling and defeated by another.\n"); + sb.append(" */\n"); + sb.append(" private static boolean wellFormedEscapes(String value) {\n"); + sb.append(" if (value == null) { return true; }\n"); + sb.append(" for (int i = 0 ; i < value.length() ; i++) {\n"); + sb.append(" if (value.charAt(i) != '%') { continue; }\n"); + sb.append(" if (i + 2 >= value.length()) { return false; }\n"); + sb.append(" if (hex(value.charAt(i + 1)) < 0 || hex(value.charAt(i + 2)) < 0) {\n"); + sb.append(" return false;\n"); + sb.append(" }\n"); + sb.append(" i += 2;\n"); + sb.append(" }\n"); + sb.append(" return true;\n"); + sb.append(" }\n\n"); + sb.append(" private static int hex(char c) {\n"); + sb.append(" if (c >= '0' && c <= '9') { return c - '0'; }\n"); + sb.append(" if (c >= 'a' && c <= 'f') { return c - 'a' + 10; }\n"); + sb.append(" if (c >= 'A' && c <= 'F') { return c - 'A' + 10; }\n"); + sb.append(" return -1;\n"); + sb.append(" }\n\n"); + + sb.append(" private static byte[] utf8(String value) {\n"); + sb.append(" try {\n"); + sb.append(" return value.getBytes(\"UTF-8\");\n"); + sb.append(" } catch (java.io.UnsupportedEncodingException err) {\n"); + sb.append(" return value.getBytes();\n"); + sb.append(" }\n"); + sb.append(" }\n\n"); + sb.append(" private static String orDefault(String value, String fallback) {\n"); + sb.append(" return value == null ? fallback : value;\n"); + sb.append(" }\n\n"); + + // The numeric binders. A malformed value takes the default rather than + // failing the request: a query string is user input, and 400 for "?page=x" + // is a choice the handler should get to make. + String[][] numeric = { + {"Int", "int", "Integer.parseInt"}, + {"Long", "long", "Long.parseLong"}, + {"Double", "double", "Double.parseDouble"}, + {"Float", "float", "Float.parseFloat"}, + {"Short", "short", "Short.parseShort"}, + {"Byte", "byte", "Byte.parseByte"}, + }; + for (int i = 0; i < numeric.length; i++) { + sb.append(" private static ").append(numeric[i][1]).append(" to") + .append(numeric[i][0]).append("(String value, ").append(numeric[i][1]) + .append(" fallback) {\n"); + sb.append(" if (value == null || value.length() == 0) {\n"); + sb.append(" return fallback;\n"); + sb.append(" }\n"); + sb.append(" try {\n"); + sb.append(" return ").append(numeric[i][2]).append("(value.trim());\n"); + sb.append(" } catch (NumberFormatException err) {\n"); + sb.append(" return fallback;\n"); + sb.append(" }\n"); + sb.append(" }\n\n"); + // The companion the guard uses. to answers the fallback for a + // value that is absent AND for one that is malformed, which is exactly + // the distinction a caller needs to make: defaultValue is documented as + // "used when the request omits it", and "zz" is not an omission. + sb.append(" private static boolean parses").append(numeric[i][0]) + .append("(String value) {\n"); + // ABSENT is fine; present and EMPTY is not. "?count=" is a parameter + // the client sent, and queryParam distinguishes it from one that was + // omitted -- so treating the two alike let an empty value take the + // default or zero and call the handler with a number nobody sent, + // which is the same defect as accepting "zz". A default is documented + // as "used when the request omits it", and this is not an omission. + sb.append(" if (value == null) {\n"); + sb.append(" return true;\n"); + sb.append(" }\n"); + sb.append(" if (value.length() == 0) {\n"); + sb.append(" return false;\n"); + sb.append(" }\n"); + sb.append(" try {\n"); + if ("Double".equals(numeric[i][0])) { + // parseDouble does not fail on a value too large for a double + // either: 1e999 comes back as infinity. The handler then runs on + // an infinite amount, and if it is written back out Json turns it + // into null, so the client is answered with neither its value nor + // an error. Float was fixed for this and Double left, which is the + // same defect one type over. + sb.append(" double asDouble = Double.parseDouble(value.trim());\n"); + sb.append(" return !Double.isInfinite(asDouble)" + + " || value.trim().indexOf(\"Infinity\") >= 0;\n"); + } else if ("Float".equals(numeric[i][0])) { + // Float.parseFloat does not FAIL on a value too large for a + // float: it answers infinity, so 1e100 passed this guard and the + // handler ran on a number the client never sent. Every other + // width throws. An input that really spells an infinity is still + // accepted, which is what parseFloat means by it. + // The SPELLING decides whether an infinity was meant, not the + // parsed value: Double.parseDouble("1e999") is itself infinite, + // so testing the parsed double declared every double-overflowing + // value to be a deliberate infinity and handed it on. The double + // guard above already tests the text; these two now agree. + sb.append(" double asDouble = Double.parseDouble(value.trim());\n"); + sb.append(" return !Float.isInfinite((float)asDouble)" + + " || value.trim().indexOf(\"Infinity\") >= 0;\n"); + } else { + sb.append(" ").append(numeric[i][2]).append("(value.trim());\n"); + sb.append(" return true;\n"); + } + sb.append(" } catch (NumberFormatException err) {\n"); + sb.append(" return false;\n"); + sb.append(" }\n"); + sb.append(" }\n\n"); + } + sb.append(" private static boolean toBoolean(String value, boolean fallback) {\n"); + sb.append(" if (value == null || value.length() == 0) {\n"); + sb.append(" return fallback;\n"); + sb.append(" }\n"); + sb.append(" // Case folding a token with toLowerCase() is locale sensitive and\n"); + sb.append(" // wrong on a Turkish device; equalsIgnoreCase is not.\n"); + sb.append(" return value.equalsIgnoreCase(\"true\") || value.equals(\"1\")\n"); + sb.append(" || value.equalsIgnoreCase(\"yes\") || value.equalsIgnoreCase(\"on\");\n"); + sb.append(" }\n\n"); + + // toBoolean answers false for everything it does not recognise, so + // "?enabled=treu" reached the handler as an explicit false and neither + // side could tell -- while the same typo in a numeric binding is a 400. + // The permissive spellings stay; what is refused is a value that is + // neither true nor false in any of them. The @RestClient half refuses + // its own malformed booleans, and the two generators disagreeing about + // the same request is its own bug. + // Empty is not false, for the reason the numeric guards already give: + // "?enabled=" is a parameter the client SENT, and binding it to false + // hands the controller a decision nobody made. Only an absent value + // takes the default. + sb.append(" private static boolean parsesBoolean(String value) {\n"); + sb.append(" if (value == null) {\n"); + sb.append(" return true;\n"); + sb.append(" }\n"); + sb.append(" if (value.length() == 0) {\n"); + sb.append(" return false;\n"); + sb.append(" }\n"); + sb.append(" return value.equalsIgnoreCase(\"true\") || value.equals(\"1\")\n"); + sb.append(" || value.equalsIgnoreCase(\"yes\") || value.equalsIgnoreCase(\"on\")\n"); + sb.append(" || value.equalsIgnoreCase(\"false\") || value.equals(\"0\")\n"); + sb.append(" || value.equalsIgnoreCase(\"no\") || value.equalsIgnoreCase(\"off\");\n"); + sb.append(" }\n\n"); + + sb.append(" private static java.util.Map bodyAsMap(String body) {\n"); + sb.append(" if (body == null || body.length() == 0) {\n"); + sb.append(" return null;\n"); + sb.append(" }\n"); + sb.append(" try {\n"); + sb.append(" return com.codename1.backend.Json.parseObject(body);\n"); + sb.append(" } catch (java.io.IOException err) {\n"); + sb.append(" return null;\n"); + sb.append(" }\n"); + sb.append(" }\n\n"); + sb.append(" private static java.util.List bodyAsList(String body) {\n"); + sb.append(" if (body == null || body.length() == 0) {\n"); + sb.append(" return null;\n"); + sb.append(" }\n"); + sb.append(" try {\n"); + sb.append(" Object parsed = com.codename1.backend.Json.parse(body);\n"); + sb.append(" // Never a cast: ParparVM's CHECKCAST is unchecked, so a wrong\n"); + sb.append(" // one reads the next instruction's fields out of the wrong\n"); + sb.append(" // object instead of throwing.\n"); + sb.append(" return parsed instanceof java.util.List ? (java.util.List)parsed : null;\n"); + sb.append(" } catch (java.io.IOException err) {\n"); + sb.append(" return null;\n"); + sb.append(" }\n"); + sb.append(" }\n"); + } + + /** + * The `main` the developer no longer writes. + * + * This is the half of the old API that was an implementation detail wearing a + * user's clothes: every server opened with the same twenty lines -- read PORT, + * start, install a shutdown handler, await termination -- and getting any of + * them wrong produced a server that leaked connections on SIGTERM or exited + * silently the moment main returned. Generating it means the controller is the + * only thing anyone writes, and the lifecycle is the same in every project. + */ + private String generateBootstrap(String packageName) { + StringBuilder sb = new StringBuilder(); + if (packageName.length() > 0) { + sb.append("package ").append(packageName).append(";\n\n"); + } + sb.append("// Generated from the @RestController classes in this module. Do not edit.\n"); + sb.append("@com.codename1.backend.annotations.Generated\n"); + sb.append("public final class BackendApplication {\n\n"); + sb.append(" private BackendApplication() {\n }\n\n"); + sb.append(" public static void main(String[] args) throws Exception {\n"); + sb.append(" com.codename1.backend.Signals.installShutdownHandler();\n"); + sb.append(" int port = 8080;\n"); + sb.append(" String configured = System.getenv(\"PORT\");\n"); + sb.append(" if (configured != null && configured.length() > 0) {\n"); + sb.append(" try {\n"); + sb.append(" port = Integer.parseInt(configured.trim());\n"); + sb.append(" } catch (NumberFormatException err) {\n"); + sb.append(" throw new IllegalStateException(\"PORT is not a number: \"\n"); + sb.append(" + configured);\n"); + sb.append(" }\n"); + sb.append(" }\n"); + sb.append(" final com.codename1.backend.HttpServer.Handler[] routers =\n"); + sb.append(" new com.codename1.backend.HttpServer.Handler[] {\n"); + int index = 0; + for (Controller c : controllers.values()) { + sb.append(" new ").append(qualify(c.packageName, c.routerSimpleName)) + .append("(new ").append(c.binaryName).append("())"); + sb.append(++index < controllers.size() ? ",\n" : "\n"); + } + sb.append(" };\n"); + sb.append(" final com.codename1.backend.HttpServer server =\n"); + sb.append(" com.codename1.backend.HttpServer.start(null, port, 512, 16,\n"); + sb.append(" new com.codename1.backend.HttpServer.Handler() {\n"); + sb.append(" public com.codename1.backend.HttpServer.Response handle(\n"); + sb.append(" com.codename1.backend.HttpServer.Request request)\n"); + sb.append(" throws Exception {\n"); + sb.append(" for (int i = 0 ; i < routers.length ; i++) {\n"); + sb.append(" com.codename1.backend.HttpServer.Response response =\n"); + sb.append(" routers[i].handle(request);\n"); + sb.append(" if (response != null) {\n"); + sb.append(" return response;\n"); + sb.append(" }\n"); + sb.append(" }\n"); + sb.append(" return null;\n"); + sb.append(" }\n"); + sb.append(" }, null);\n"); + sb.append(" com.codename1.backend.Signals.onShutdown(new Runnable() {\n"); + sb.append(" public void run() {\n"); + sb.append(" // Stop accepting and let what is in flight finish.\n"); + sb.append(" // Signals ends the process; exiting from here would\n"); + sb.append(" // deadlock the JVM shutdown hook this runs from.\n"); + sb.append(" server.stop(10000);\n"); + sb.append(" }\n"); + sb.append(" });\n"); + sb.append(" // Required: the host threads are detached, so a main that returned\n"); + sb.append(" // would end the process without a word.\n"); + sb.append(" server.awaitTermination();\n"); + sb.append(" }\n"); + sb.append("}\n"); + return sb.toString(); + } + + /** A route as a byte[] constant, which is what the request is compared against. */ + private static String byteArrayLiteral(String value) { + StringBuilder sb = new StringBuilder("{"); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c > 0x7f) { + // A route pattern is written in source, and a non-ASCII one would have + // to be compared against its percent-encoded form on the wire. + throw new IllegalArgumentException("Route patterns must be ASCII: " + value); + } + if (i > 0) { + sb.append(", "); + } + sb.append((int) c); + } + return sb.append("}").toString(); + } + + private static String stringArrayLiteral(List values) { + StringBuilder sb = new StringBuilder("{"); + for (int i = 0; i < values.size(); i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(quote(values.get(i))); + } + return sb.append("}").toString(); + } + + private static String quote(String value) { + StringBuilder sb = new StringBuilder("\""); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c == '"' || c == '\\') { + sb.append('\\').append(c); + } else if (c == '\n') { + sb.append("\\n"); + } else if (c == '\r') { + sb.append("\\r"); + } else if (c < 0x20 || c > 0x7e) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + return sb.append('"').toString(); + } +} diff --git a/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestServerAnnotationProcessor.java b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestServerAnnotationProcessor.java new file mode 100644 index 00000000000..da3aeb5d847 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/processors/RestServerAnnotationProcessor.java @@ -0,0 +1,1824 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven.processors; + +import com.codename1.maven.annotations.AbstractAnnotationProcessor; +import com.codename1.maven.annotations.AnnotatedClass; +import com.codename1.maven.annotations.AnnotationValues; +import com.codename1.maven.annotations.FieldInfo; +import com.codename1.maven.annotations.JavaSourceCompiler; +import com.codename1.maven.annotations.MethodInfo; +import com.codename1.maven.annotations.ProcessingException; +import com.codename1.maven.annotations.ProcessorContext; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.objectweb.asm.Type; + +/// Server-side half of the `@RestClient` contract. +/// +/// The SAME annotated interface that +/// [RestClientAnnotationProcessor] turns into a typed client is turned here into +/// the two things a server needs. One declaration, both ends, so a change to the +/// contract is a compile error on whichever side did not follow it -- which is the +/// entire point of sharing the interface rather than hand-writing a client against +/// a REST endpoint. +/// +/// For `@RestClient interface GreeterApi` it emits, in the interface's package: +/// +/// 1. `GreeterApiServer` -- a SYNCHRONOUS interface the backend implements. The +/// client's methods are asynchronous (they take an +/// `OnComplete>` and return void); a server handler has nothing to +/// do with a callback, so its method returns `T` directly and the callback +/// parameter is dropped. This is gRPC's shape: one definition, an async client +/// stub and a sync server base, rather than forcing one signature to serve both. +/// It is also what keeps `Response` off the server's classpath entirely -- its +/// constructor is package-private, so server code could not build one anyway. +/// 2. `GreeterApiDispatcher` -- routes `(method, path, body)` to the right handler +/// method, binding `@Path` segments and `@Query` parameters out of the request. +/// +/// Unsupported bindings are rejected with an error rather than silently bound to +/// null: a parameter that quietly arrives empty at runtime is far more expensive +/// than a build that refuses to produce one. +public final class RestServerAnnotationProcessor extends AbstractAnnotationProcessor { + + private static final Set DESCRIPTORS; + static { + Set s = new LinkedHashSet(); + s.add(RestClientAnnotationProcessor.REST_CLIENT_DESC); + DESCRIPTORS = Collections.unmodifiableSet(s); + } + + /// Set -Dcn1.restServer=true (or the cn1.restServer property) to emit the + /// server half. Off by default: every existing project carries @RestClient + /// interfaces for its client, and generating server classes into those builds + /// would grow every app for nothing. + static boolean isEnabled() { + return "true".equalsIgnoreCase(System.getProperty("cn1.restServer", "false")); + } + + private final TreeMap accepted = new TreeMap(); + + /// DTO types reachable from an accepted contract, keyed by binary name. A + /// TreeMap so codec emission order is stable across builds. + private final TreeMap dtos = new TreeMap(); + + static final class Api { + String binaryName, simpleName, packageName, serverSimpleName, dispatcherSimpleName; + final List ops = new ArrayList(); + } + + static final class Op { + String name, verb, pathTemplate, returnType; + final List params = new ArrayList(); + } + + static final class Param { + String javaType, name, bindKind, bindName; + } + + @Override + public Set getAnnotationDescriptors() { + return DESCRIPTORS; + } + + @Override + public void start(ProcessorContext ctx) throws ProcessingException { + accepted.clear(); + dtos.clear(); + } + + @Override + public void processClass(AnnotatedClass cls, ProcessorContext ctx) throws ProcessingException { + if (!isEnabled()) return; + if (cls.isSynthetic()) return; + if (cls.getClassAnnotation(RestClientAnnotationProcessor.REST_CLIENT_DESC) == null) return; + // Shape errors (not an interface, not public) are already reported by the + // client processor over the same class; repeating them would double every + // message in the build log. + if (!cls.isInterface() || !cls.isPublic()) return; + + Api api = new Api(); + api.binaryName = cls.getBinaryName(); + api.simpleName = RestClientAnnotationProcessor.simpleName(api.binaryName); + api.packageName = RestClientAnnotationProcessor.packageOf(api.binaryName); + api.serverSimpleName = api.simpleName + "Server"; + api.dispatcherSimpleName = api.simpleName + "Dispatcher"; + + boolean anyError = false; + for (MethodInfo m : cls.getMethods()) { + if (m.isStatic() || m.isSynthetic() || m.isConstructor() || !m.isAbstract()) continue; + if ((m.getAccess() & org.objectweb.asm.Opcodes.ACC_BRIDGE) != 0) continue; + + Op op = new Op(); + op.name = m.getName(); + + AnnotationValues va; + int verbCount = 0; + if ((va = m.getAnnotation(RestClientAnnotationProcessor.GET_DESC)) != null) { op.verb = "GET"; op.pathTemplate = va.getString("value"); verbCount++; } + if ((va = m.getAnnotation(RestClientAnnotationProcessor.POST_DESC)) != null) { op.verb = "POST"; op.pathTemplate = va.getString("value"); verbCount++; } + if ((va = m.getAnnotation(RestClientAnnotationProcessor.PUT_DESC)) != null) { op.verb = "PUT"; op.pathTemplate = va.getString("value"); verbCount++; } + if ((va = m.getAnnotation(RestClientAnnotationProcessor.DELETE_DESC)) != null) { op.verb = "DELETE"; op.pathTemplate = va.getString("value"); verbCount++; } + if ((va = m.getAnnotation(RestClientAnnotationProcessor.PATCH_DESC)) != null) { op.verb = "PATCH"; op.pathTemplate = va.getString("value"); verbCount++; } + if (verbCount != 1) continue; // the client processor reports this + if (op.pathTemplate == null) op.pathTemplate = ""; + + Type[] paramTypes = Type.getArgumentTypes(m.getDescriptor()); + List> paramAnnotations = m.getParameterAnnotations(); + String[] genericSigs = RestClientAnnotationProcessor + .parseGenericParameterSignatures(m.getSignature(), paramTypes.length); + + op.returnType = "void"; + int bodyCount = 0; + for (int i = 0; i < paramTypes.length; i++) { + String descriptor = paramTypes[i].getDescriptor(); + String genericSig = genericSigs == null ? null : genericSigs[i]; + + if (RestClientAnnotationProcessor.isCallbackType(descriptor)) { + // The callback is the client's result channel. On the server it + // becomes the return type and disappears from the signature. + String payload = RestClientAnnotationProcessor.extractResponsePayload(genericSig); + op.returnType = (payload == null || payload.length() == 0) + ? "java.lang.Object" : payload; + collectDtos(op.returnType, ctx); + continue; + } + + Map pa = i < paramAnnotations.size() ? paramAnnotations.get(i) : null; + Param p = new Param(); + p.javaType = RestClientAnnotationProcessor.javaTypeFor(paramTypes[i], genericSig); + AnnotationValues bind; + if (pa != null && (bind = pa.get(RestClientAnnotationProcessor.PATH_DESC)) != null) { + p.bindKind = "path"; + p.bindName = bind.getString("value"); + } else if (pa != null && (bind = pa.get(RestClientAnnotationProcessor.QUERY_DESC)) != null) { + p.bindKind = "query"; + p.bindName = bind.getString("value"); + } else if (pa != null && (bind = pa.get(RestClientAnnotationProcessor.HEADER_DESC)) != null) { + p.bindKind = "header"; + p.bindName = bind.getString("value"); + } else if (pa != null && (bind = pa.get(RestClientAnnotationProcessor.COOKIE_DESC)) != null) { + p.bindKind = "cookie"; + p.bindName = bind.getString("value"); + } else if (pa != null && pa.get(RestClientAnnotationProcessor.BODY_DESC) != null) { + p.bindKind = "body"; + p.bindName = "body"; + bodyCount++; + } else { + ctx.error(cls, "Parameter " + i + " of " + api.binaryName + "." + op.name + + " carries no REST binding annotation, so the dispatcher cannot " + + "supply a value for it"); + anyError = true; + continue; + } + if (p.bindName == null || p.bindName.length() == 0) p.bindName = "p" + i; + p.name = RestClientAnnotationProcessor.sanitizeIdentifier( + "body".equals(p.bindKind) ? "body" : p.bindName); + if ("body".equals(p.bindKind)) { + collectDtos(p.javaType, ctx); + } else if (!isBindableScalar(p.javaType)) { + // Path/query/header/cookie values arrive as text. Anything that + // is not convertible from a String has no defined binding, and + // guessing one would silently hand the handler a null. + ctx.error(cls, "Parameter " + i + " of " + api.binaryName + "." + op.name + + " is bound from the request as text but has type " + p.javaType + + ", which cannot be parsed from a string; use @Body for structured input"); + anyError = true; + continue; + } + op.params.add(p); + } + if (bodyCount > 1) { + ctx.error(cls, api.binaryName + "." + op.name + + " declares more than one @Body parameter; a request has one body"); + anyError = true; + } + // Every @Path has to name a placeholder that is actually in the template. + // A typo bound null, or 0 for a primitive, and the route still matched -- + // so the handler ran with the wrong identifier and nothing said so. + String[] template = splitTemplate(op.pathTemplate); + for (int pi = 0; pi < op.params.size(); pi++) { + Param p = op.params.get(pi); + if ("path".equals(p.bindKind) && placeholderIndex(template, p.bindName) < 0) { + ctx.error(cls, api.binaryName + "." + op.name + " binds @Path(\"" + + p.bindName + "\") but the route " + op.pathTemplate + + " has no {" + p.bindName + "} to bind it to"); + anyError = true; + } + } + // And the other direction, which is the half that was missing. A + // placeholder nothing binds is worse than a typo: the CLIENT + // substitutes the placeholder's own name, so it requests /users/id + // literally, while the SERVER matches any value there and passes it + // to nobody. Both halves compile, and the route they agree on is one + // whose variable cannot be supplied or read. + for (int ti = 0; ti < template.length; ti++) { + String name = placeholderName(template[ti]); + if (name == null) { + continue; + } + if (hasSecondPlaceholder(template[ti])) { + // Said out loud rather than matched approximately. "{a}-{b}" + // has no single reading -- where one value ends and the next + // begins is a guess -- and a server that guesses binds + // something the client never meant. The client half accepts + // this shape, so the developer is told where the disagreement + // is instead of meeting a route that never matches. + ctx.error(cls, api.binaryName + "." + op.name + " declares the route " + + op.pathTemplate + ", whose segment '" + template[ti] + + "' holds more than one placeholder. Where one value ends " + + "and the next begins cannot be decided from the path, so " + + "give each placeholder its own segment."); + anyError = true; + continue; + } + boolean bound = false; + for (int pi = 0; pi < op.params.size(); pi++) { + Param p = op.params.get(pi); + if ("path".equals(p.bindKind) && name.equals(p.bindName)) { + bound = true; + break; + } + } + if (!bound) { + ctx.error(cls, api.binaryName + "." + op.name + " declares the route " + + op.pathTemplate + ", but nothing binds {" + name + "}. Add a " + + "parameter annotated @Path(\"" + name + "\"), or take the " + + "placeholder out of the path."); + anyError = true; + } + } + api.ops.add(op); + } + // Two routes of the same verb and shape generate the same predicate, and + // dispatch takes the first that matches -- so the second is unreachable + // however it is called. The names differ; the SHAPE is what the router sees. + Map shapes = new LinkedHashMap(); + for (Op op : api.ops) { + String shape = op.verb + " " + placeholderShape(op.pathTemplate); + String first = shapes.get(shape); + if (first != null) { + ctx.error(cls, api.binaryName + "." + op.name + " and " + first + + " are both " + shape + " once the placeholder names are" + + " taken out, so only the first can ever be reached"); + anyError = true; + continue; + } + // Equality is not the only way two routes collide. "/a/{x}/c" and + // "/a/b/{y}" are different shapes and BOTH answer /a/b/c: a placeholder + // takes any value in its segment, so two dynamic patterns can overlap + // without either being more specific. Literal-first ordering cannot + // break that tie because neither is literal, and dispatch returns from + // whichever it emits first, so the contract gives that request no + // stable meaning. + String clash = overlappingShape(shapes.keySet(), shape); + // Unless one of the two is wholly literal. The dispatcher emits every + // route without a placeholder before every route with one, so + // "GET /users/me" beside "GET /users/{id}" is decided by that order: + // the literal takes its own path and every other value falls through. + // The comment above is right that literal-first cannot break a tie + // between two DYNAMIC shapes -- and equally, it does break this one. + if (clash != null && isLiteralShape(clash) != isLiteralShape(shape)) { + clash = null; + } + if (clash != null) { + ctx.error(cls, api.binaryName + "." + op.name + " answers " + shape + + ", which " + shapes.get(clash) + " also answers as " + clash + + ". A path satisfying both is dispatched to whichever comes " + + "first, so give them different paths."); + anyError = true; + continue; + } + shapes.put(shape, op.name); + } + if (!anyError && !api.ops.isEmpty()) { + accepted.put(api.binaryName, api); + } + } + + /// A route with its placeholder NAMES removed, which is all the generated + /// router matches on: "/pets/{id}" and "/pets/{name}" are one shape. + /** The descriptor the scanner keys @Generated by. */ + private static final String GENERATED = "Lcom/codename1/backend/annotations/Generated;"; + + private static String placeholderShape(String template) { + String[] parts = splitTemplate(template); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < parts.length; i++) { + // The LITERALS around a placeholder are part of the shape. Collapsing + // the whole segment to "{}" made "/{name}.json" and "/{name}.xml" the + // same shape, so the duplicate check refused a pair that no single + // request can satisfy -- and the matcher supports them now, which is + // what makes the refusal wrong rather than conservative. + if (isPlaceholder(parts[i])) { + sb.append('/').append(placeholderPrefix(parts[i])).append("{}") + .append(placeholderSuffix(parts[i])); + } else { + sb.append('/').append(parts[i]); + } + } + return sb.length() == 0 ? "/" : sb.toString(); + } + + /// Records any application class reachable as a body or a result so a codec is + /// emitted for it. `java.util.List` contributes Foo, not List. + /** + * A DTO whose public fields the generated codec can actually round-trip. + * + * The encoder writes every public field; the decoder assigns them after a + * no-argument construction, so a FINAL one is written and then silently not + * read back. The handler gets the initializer and the client's value is gone, + * with nothing failing at build time or at request time to say so. Refused + * here instead: the contract cannot be honoured, so it should not compile. + */ + private void requireAssignableFields(String binaryName, AnnotatedClass cls, + ProcessorContext ctx) { + for (FieldInfo f : transferredFields(cls, ctx)) { + if (f.isFinal()) { + ctx.error(cls, binaryName + "." + f.getName() + " is public and final, " + + "so the generated decoder cannot assign it: the field would " + + "be sent by the client and silently dropped on arrival. Drop " + + "the final, or keep the field out of the transferred shape."); + } + } + } + + /** + * Refuses to generate over a class the project already has. + * + * What is compiled here lands in the same output directory, so a name that + * already exists is simply overwritten -- silently, because what is generated + * compiles perfectly well. Every family generated here needs this, not just + * the first one somebody thought of: the server interface, the dispatcher and + * each DTO codec are all derived names a developer could have used. + */ + private boolean wouldReplaceAnExistingClass(String binaryName, String what, + ProcessorContext ctx) { + AnnotatedClass existing = ctx.lookup(binaryName.replace('.', '/')); + if (existing == null) { + return false; + } + if (existing.getClassAnnotations().containsKey(GENERATED)) { + // Our own output from an earlier pass. An incremental build scans + // target/classes, so without this an unchanged contract could be + // processed exactly once per clean -- the second run reported the + // ApiServer, ApiDispatcher and every DTO codec as existing + // application classes. A genuine collision carries no marker. + return false; + } + ctx.error(binaryName + " already exists, and the " + what + " generated for " + + "this API would replace it. Rename that class, or rename the " + + "interface the name is derived from."); + return true; + } + + /** + * Every public instance field a DTO carries, its superclasses included. + * + * AnnotatedClass.getFields() reads ONE class file, so an inherited field was + * invisible to all four passes that use it: it was not validated, its type was + * never collected, the encoder never wrote it and the decoder never read it. A + * subclass went over the wire missing everything its base declared, silently + * and on both ends. Walks up until the superclass is outside the index, which + * is where the JDK begins; a field hidden by one of the same name in a subclass + * is taken from the subclass, as Java resolves it. + */ + /** The already-seen shape that a path could satisfy along with this one, or null. */ + private static String overlappingShape(Set seen, String shape) { + for (String other : seen) { + if (shapesOverlap(other, shape)) { + return other; + } + } + return null; + } + + /** Same verb, same segment count, and every pair of segments compatible. */ + private static boolean shapesOverlap(String left, String right) { + int leftSpace = left.indexOf(' '); + int rightSpace = right.indexOf(' '); + if (leftSpace < 0 || rightSpace < 0 + || !left.substring(0, leftSpace).equals(right.substring(0, rightSpace))) { + return false; + } + String[] a = left.substring(leftSpace + 1).split("/", -1); + String[] b = right.substring(rightSpace + 1).split("/", -1); + if (a.length != b.length) { + return false; + } + for (int i = 0; i < a.length; i++) { + // ONE implementation of this rule, in the controller processor. There + // used to be two, and they disagreed: that one learned that "{}.json" + // and "{}.xml" cannot both match while this one still called every pair + // of variable-carrying segments a collision, so a contract the matcher + // handles was refused here and accepted there. A rule copied is a rule + // that will be fixed once. + if (!RestControllerAnnotationProcessor.segmentsOverlap(a[i], b[i])) { + return false; + } + } + return true; + } + + private List transferredFields(AnnotatedClass cls, ProcessorContext ctx) { + List out = new ArrayList(); + Set seen = new LinkedHashSet(); + AnnotatedClass at = cls; + while (at != null) { + for (FieldInfo f : at.getFields()) { + if (f.isStatic() || !f.isPublic()) { + continue; + } + if ((f.getAccess() & org.objectweb.asm.Opcodes.ACC_SYNTHETIC) != 0) { + continue; + } + if (seen.add(f.getName())) { + out.add(f); + } + } + String superName = at.getSuperInternalName(); + at = superName == null ? null : ctx.lookup(superName); + } + return out; + } + + private void collectDtos(String javaType, ProcessorContext ctx) { + if (javaType == null) return; + String t = javaType.trim(); + int lt = t.indexOf('<'); + if (lt >= 0) { + String outer = t.substring(0, lt); + String inner = t.substring(lt + 1, t.length() - 1); + if ("java.util.List".equals(outer) || "java.util.Set".equals(outer) + || "java.util.Collection".equals(outer)) { + // A collection OF a collection of DTOs encodes wrongly and quietly: + // fieldToJson applies the generated codec to the elements of the + // outer collection only, and an element that is itself a collection + // starts with "java." so it is handed to the writer untouched -- + // where each DTO inside becomes the JSON string of its toString(). + // Refused for the same reason a Map of DTOs is: the codec cannot + // express the shape, and producing the wrong JSON is worse than + // refusing to compile. + if (inner.indexOf('<') >= 0 && namesADto(inner, ctx)) { + ctx.error("A transferred field or return typed " + t + " cannot be " + + "encoded: the generated codec reaches the elements of the " + + "outer collection only, so the DTOs inside " + inner + + " would be written as their toString(). Use a collection " + + "of a DTO that holds the inner collection."); + return; + } + collectDtos(inner, ctx); + } else if ("java.util.Map".equals(outer)) { + // A Map of JDK values round-trips; a Map whose values are a DTO + // does not, and does so QUIETLY. Only List and Set recursed here, + // so no codec was generated for the value type: the decoder does a + // guarded Map cast and leaves a decoded Map in a field declared as + // that DTO, and the encoder writes its toString() as a JSON string. + // Both halves compile and neither works, so the shape is refused + // rather than mistranslated. Generating conversions for it is a + // feature, not a fix for this. + // The KEY as well. A JSON object's member names are strings, always, + // so Map receives string keys under an integer-keyed + // declaration: a get(Integer) finds nothing and iterating the entries + // as Integer throws, while encoding turns the integers back into + // strings. Only a String key round-trips. + String key = mapKeyType(inner); + String rawKey = key.indexOf('<') < 0 ? key : key.substring(0, key.indexOf('<')); + if (!"java.lang.String".equals(rawKey) && !"java.lang.Object".equals(rawKey)) { + ctx.error("A transferred field or return typed " + t + " cannot be " + + "decoded: a JSON object's names are strings, so " + rawKey + + " keys arrive as String and neither lookup nor iteration " + + "works. Key the map by String."); + return; + } + String value = mapValueType(inner); + // A Map's VALUES are handed over exactly as the parser made them: + // unlike a List or a Set, nothing walks them applying the element + // type. The parser answers Long for every JSON integer and Double + // for every real, so Map is a map of Long at + // runtime -- the cast erases, and the handler's first read as an + // Integer throws. Declaring Long or Double says what actually + // arrives; the numeric types that need converting do not. + String rawValue = value.indexOf('<') < 0 ? value + : value.substring(0, value.indexOf('<')); + // The raw type is not the whole answer: Map> + // has an acceptable OUTER value and an Integer inside it that the + // parser never produces. Nothing converts a map's values at any + // depth, so every level has to be a type that arrives as itself. + if (!namesADto(value, ctx) && !mapValueArrivesAsDeclared(value)) { + ctx.error("A transferred field or return typed " + t + " cannot be " + + "decoded: a map's values arrive as the parser built them, so " + + value + " would really be " + parserTypeFor(rawValue) + + " and reading it as " + rawValue + " throws. Use a Map of " + + "Long, Double, Boolean, String, Map or List, or a DTO."); + return; + } + if (namesADto(value, ctx)) { + ctx.error("A transferred field typed " + t + " cannot be encoded: " + + "the generated codec round-trips a Map of JDK values " + + "only, and " + value + " would be silently replaced by " + + "a plain Map on the way in. Use a list of a DTO that " + + "carries the key, or a Map with JDK value types."); + } + } + return; + } + if (t.indexOf('.') < 0) return; // a primitive + if (t.startsWith("java.")) { + // Not every JDK type round-trips, and the ones that do not fail + // SILENTLY in both directions. A field typed java.util.Date is the + // plain case: the client sends a number, so the decoder's guarded + // cast to Date never matches and the field arrives null, while the + // encoder hands the Date to Json and gets its toString() -- the + // contract compiles at both ends and the value survives neither. + // The same is true of BigDecimal, UUID and every java.time type, so + // the answer is the supported set rather than a case for Date. + if (!CODEC_JDK_TYPES.contains(t)) { + ctx.error("A transferred field or return typed " + t + " cannot be " + + "encoded: the generated codec handles the primitives and their " + + "boxes, String, byte[], and List, Set or Map of those. " + t + + " would arrive null and be written as its toString(). Carry it " + + "as a long of epoch milliseconds or as a String."); + } + return; + } + AnnotatedClass cls = ctx.lookup(t.replace('.', '/')); + if (cls == null || cls.isInterface() || cls.isEnum()) return; + if (dtos.containsKey(t)) return; + requireAssignableFields(t, cls, ctx); + dtos.put(t, cls); + for (FieldInfo f : transferredFields(cls, ctx)) { + collectDtos(fieldJavaType(f), ctx); + } + } + + /** + * The JDK types a generated codec can convert in BOTH directions. Taken from + * the branches of the conversion above, plus the containers handled by the + * generic path and byte[]; a type added there belongs here too. Anything else + * under java.* reaches the guarded cast, which cannot match a value the JSON + * parser produced. + */ + private static final Set CODEC_JDK_TYPES = Collections.unmodifiableSet( + new LinkedHashSet(Arrays.asList( + "java.lang.String", "java.lang.Integer", "java.lang.Long", + "java.lang.Double", "java.lang.Boolean", "java.lang.Float", + "java.lang.Short", "java.lang.Byte", + "java.util.List", "java.util.Set", "java.util.Collection", + "java.util.Map"))); + + /** A shape with no placeholder at all, which the dispatcher emits first. */ + private static boolean isLiteralShape(String shape) { + return shape.indexOf("{}") < 0; + } + + /** + * What a Map's values may be declared as, which is exactly what Json.parse + * produces: it answers Long for every JSON integer and Double for every real, + * regardless of how the field is declared, and nothing converts a map's + * values afterwards the way collection elements are converted. + */ + private static final Set PARSED_MAP_VALUE_TYPES = Collections.unmodifiableSet( + new LinkedHashSet(Arrays.asList( + "java.lang.Object", "java.lang.String", "java.lang.Long", + "java.lang.Double", "java.lang.Boolean", + // Map and List only. Set and Collection are NOT here even + // though a collection field elsewhere may be declared as + // either: a JSON array always arrives as a List, and the + // element conversion that turns one into a Set runs for + // FIELDS, never for a map's values -- so Map> + // hands the handler a List under a Set declaration and throws + // on first use. What a map's value may be declared as is + // exactly what the parser hands over, with nothing in between. + "java.util.Map", "java.util.List"))); + + /** What the parser really answers where the declared type says otherwise. */ + private static String parserTypeFor(String declared) { + if ("java.lang.Integer".equals(declared) || "java.lang.Short".equals(declared) + || "java.lang.Byte".equals(declared)) { + return "a Long"; + } + if ("java.lang.Float".equals(declared)) { + return "a Double"; + } + return "something else"; + } + + /** + * A declared collection this codec converts element by element. Collection + * belongs with List and Set: the parser answers an ArrayList either way, so + * a Collection that is NOT recognised here falls through to a guarded + * cast, which erases -- the handler is then holding a collection of Map under + * a Collection declaration and throws on its first element. The three + * have to be listed everywhere any of them is, which is why this is one + * method rather than three copies of the same disjunction. + */ + private static boolean isCollectionShape(String javaType) { + return javaType.startsWith("java.util.List<") + || javaType.startsWith("java.util.Set<") + || javaType.startsWith("java.util.Collection<"); + } + + /** + * Whether a map value's declared type is what the parser really hands over, + * all the way down. A map's values are never converted -- not at the top + * level and not inside a nested container -- so each level must already be + * what arrives: Map, List, String, Long, Double, Boolean or Object. + */ + private static boolean mapValueArrivesAsDeclared(String javaType) { + int lt = javaType.indexOf('<'); + String raw = lt < 0 ? javaType : javaType.substring(0, lt); + if (!PARSED_MAP_VALUE_TYPES.contains(raw)) { + return false; + } + if (lt < 0) { + return true; + } + int end = javaType.lastIndexOf('>'); + if (end <= lt) { + return true; + } + List args = RestControllerAnnotationProcessor.splitTypeArguments( + javaType.substring(lt + 1, end)); + for (int i = 0; i < args.size(); i++) { + String arg = args.get(i); + if (arg.startsWith("?")) { + continue; + } + if (!mapValueArrivesAsDeclared(arg)) { + return false; + } + } + return true; + } + + /** The key half of a Map's type arguments, honouring nested generics. */ + private static String mapKeyType(String inner) { + int depth = 0; + for (int i = 0; i < inner.length(); i++) { + char c = inner.charAt(i); + if (c == '<') { + depth++; + } else if (c == '>') { + depth--; + } else if (c == ',' && depth == 0) { + return inner.substring(0, i).trim(); + } + } + return inner.trim(); + } + + /** The value half of a Map's type arguments, honouring nested generics. */ + private static String mapValueType(String inner) { + int depth = 0; + for (int i = 0; i < inner.length(); i++) { + char c = inner.charAt(i); + if (c == '<') { + depth++; + } else if (c == '>') { + depth--; + } else if (c == ',' && depth == 0) { + return inner.substring(i + 1).trim(); + } + } + return inner.trim(); + } + + /** Whether a type, or anything inside its type arguments, is one of ours. */ + private static boolean namesADto(String javaType, ProcessorContext ctx) { + if (javaType == null) { + return false; + } + String[] tokens = javaType.split("[<>,]"); + for (int i = 0; i < tokens.length; i++) { + String token = tokens[i].trim(); + if (token.length() == 0 || token.startsWith("java.") || token.indexOf('.') < 0) { + continue; + } + AnnotatedClass cls = ctx.lookup(token.replace('.', '/')); + if (cls != null && !cls.isInterface() && !cls.isEnum()) { + return true; + } + } + return false; + } + + private static String fieldJavaType(FieldInfo f) { + String sig = f.getSignature(); + if (sig != null && sig.length() > 0) { + return RestClientAnnotationProcessor.jvmSignatureToJavaType(sig); + } + return RestClientAnnotationProcessor.jvmSignatureToJavaType(f.getDescriptor()); + } + + private static boolean isBindableScalar(String javaType) { + return "java.lang.String".equals(javaType) || "int".equals(javaType) || "long".equals(javaType) + || "boolean".equals(javaType) || "double".equals(javaType) || "float".equals(javaType) + || "short".equals(javaType) || "byte".equals(javaType) + || "java.lang.Integer".equals(javaType) || "java.lang.Long".equals(javaType) + || "java.lang.Boolean".equals(javaType) || "java.lang.Double".equals(javaType) + || "java.lang.Float".equals(javaType) || "java.lang.Short".equals(javaType) + || "java.lang.Byte".equals(javaType); + } + + @Override + public void finish(ProcessorContext ctx) throws ProcessingException { + if (!isEnabled() || ctx.hasErrors() || accepted.isEmpty()) return; + Map sources = new LinkedHashMap(); + for (Api api : accepted.values()) { + String server = qualify(api.packageName, api.serverSimpleName); + String dispatcher = qualify(api.packageName, api.dispatcherSimpleName); + if (wouldReplaceAnExistingClass(server, "server interface", ctx) + || wouldReplaceAnExistingClass(dispatcher, "dispatcher", ctx)) { + return; + } + sources.put(server, generateServerInterface(api)); + sources.put(dispatcher, generateDispatcher(api)); + } + for (Map.Entry e : dtos.entrySet()) { + String pkg = RestClientAnnotationProcessor.packageOf(e.getKey()); + String simple = RestClientAnnotationProcessor.simpleName(e.getKey()) + "Json"; + String codec = qualify(pkg, simple); + if (wouldReplaceAnExistingClass(codec, "JSON codec", ctx)) { + return; + } + sources.put(codec, generateDtoCodec(e.getKey(), e.getValue(), ctx)); + } + try { + List cp = new ArrayList(); + cp.add(ctx.getOutputClassDir()); + JavaSourceCompiler.compile(sources, ctx.getOutputClassDir(), cp); + } catch (IOException ioe) { + throw new ProcessingException("Could not compile generated @RestClient server sources: " + + ioe.getMessage(), ioe); + } + ctx.getLog().info("cn1: generated " + accepted.size() + " @RestClient server dispatcher(s) and " + + dtos.size() + " DTO codec(s)"); + } + + private static String qualify(String pkg, String simple) { + return pkg.length() == 0 ? simple : pkg + "." + simple; + } + + private static String codecFor(String dtoBinaryName) { + return qualify(RestClientAnnotationProcessor.packageOf(dtoBinaryName), + RestClientAnnotationProcessor.simpleName(dtoBinaryName) + "Json"); + } + + private static String generateServerInterface(Api api) { + StringBuilder sb = new StringBuilder(1024); + if (api.packageName.length() > 0) sb.append("package ").append(api.packageName).append(";\n\n"); + sb.append("// Auto-generated by cn1:process-annotations from ").append(api.binaryName).append(". Do not edit.\n"); + sb.append("@SuppressWarnings({\"all\"})\n"); + sb.append("@com.codename1.backend.annotations.Generated\n"); + sb.append("public interface ").append(api.serverSimpleName).append(" {\n"); + for (Op op : api.ops) { + sb.append(" ").append(op.returnType).append(' ').append(op.name).append('('); + for (int i = 0; i < op.params.size(); i++) { + if (i > 0) sb.append(", "); + sb.append(op.params.get(i).javaType).append(' ').append(op.params.get(i).name); + } + sb.append(") throws Exception;\n"); + } + sb.append("}\n"); + return sb.toString(); + } + + // ---------------------------------------------------------------- + // Dispatcher + // ---------------------------------------------------------------- + + private static String generateDispatcher(Api api) { + StringBuilder sb = new StringBuilder(8192); + if (api.packageName.length() > 0) sb.append("package ").append(api.packageName).append(";\n\n"); + sb.append("// Auto-generated by cn1:process-annotations from ").append(api.binaryName).append(". Do not edit.\n"); + sb.append("//\n"); + sb.append("// References nothing outside java.*, on purpose: generated server code has to\n"); + sb.append("// link into a binary that has no Codename One implementation. JSON text is\n"); + sb.append("// parsed and written by the caller, so `body` arrives as an already-decoded\n"); + sb.append("// Map/List/String and the result goes back the same way.\n"); + sb.append("@SuppressWarnings({\"all\"})\n"); + sb.append("@com.codename1.backend.annotations.Generated\n"); + sb.append("public final class ").append(api.dispatcherSimpleName).append(" {\n"); + sb.append(" private final ").append(api.serverSimpleName).append(" impl;\n\n"); + sb.append(" public ").append(api.dispatcherSimpleName).append("(") + .append(api.serverSimpleName).append(" impl) {\n this.impl = impl;\n }\n\n"); + + sb.append(" /** True when this API has a route for the verb and path. */\n"); + sb.append(" public boolean hasRoute(String method, String rawPath) {\n"); + sb.append(" String path = stripQuery(rawPath);\n"); + sb.append(" String[] seg = split(path);\n"); + for (Op op : api.ops) { + sb.append(" if(").append(routeCondition(op)).append(") return true;\n"); + } + sb.append(" return false;\n"); + sb.append(" }\n\n"); + + sb.append(" /**\n"); + sb.append(" * Invokes the handler for this request.\n"); + sb.append(" *\n"); + sb.append(" * headers may be null. body is the decoded JSON value (Map/List/String) or\n"); + sb.append(" * null. Returns a JSON-ready value; check hasRoute first, because a handler\n"); + sb.append(" * returning null and no route at all both come back as null.\n"); + sb.append(" */\n"); + sb.append(" public Object dispatch(String method, String rawPath, java.util.Map headers, Object body) throws Exception {\n"); + sb.append(" if(!wellFormedEscapes(rawPath)) {\n"); + sb.append(" throw new IllegalArgumentException(\"malformed percent-escape in the " + + "request target: \" + rawPath);\n"); + sb.append(" }\n"); + sb.append(" String path = stripQuery(rawPath);\n"); + sb.append(" String query = queryOf(rawPath);\n"); + sb.append(" String[] seg = split(path);\n"); + // Order matters HERE and nowhere else in this file: dispatch returns from + // the first branch that matches, while hasRoute is an or and the interface + // is only declarations. A route with a placeholder accepts any value in that + // segment, so "GET /notes/{id}" declared before "GET /notes/latest" answered + // /notes/latest itself and the literal method could never run. The generator + // for @RestController already sorts for this; this half did not. + List ordered = new ArrayList(api.ops); + Collections.sort(ordered, new java.util.Comparator() { + public int compare(Op a, Op b) { + int byVerb = a.verb.compareTo(b.verb); + if (byVerb != 0) { + return byVerb; + } + boolean aVar = placeholderShape(a.pathTemplate).indexOf("{}") >= 0; + boolean bVar = placeholderShape(b.pathTemplate).indexOf("{}") >= 0; + if (aVar != bVar) { + return aVar ? 1 : -1; + } + return b.pathTemplate.length() - a.pathTemplate.length(); + } + }); + for (Op op : ordered) { + emitRoute(sb, op); + } + sb.append(" return null;\n"); + sb.append(" }\n\n"); + emitHelpers(sb); + sb.append("}\n"); + return sb.toString(); + } + + private static String routeCondition(Op op) { + String[] template = splitTemplate(op.pathTemplate); + StringBuilder sb = new StringBuilder(); + sb.append('"').append(op.verb).append("\".equals(method) && seg.length == ").append(template.length); + for (int i = 0; i < template.length; i++) { + if (!isPlaceholder(template[i])) { + sb.append(" && \"").append(RestClientAnnotationProcessor.escape(template[i])) + .append("\".equals(seg[").append(i).append("])"); + } else { + // A placeholder stands for a NON-EMPTY run within its segment, and + // any literal text around it has to match too. /pets/ splits to the + // same COUNT as /pets/{id}, so without the length test the route ran + // with id set to the empty string rather than not matching -- a path + // the contract does not describe. The overlap checker models a + // placeholder as [^/]+ and the @RestController router refuses an + // empty variable, so this is the rule the rest of the system already + // applies. + String prefix = placeholderPrefix(template[i]); + String suffix = placeholderSuffix(template[i]); + if (prefix.length() > 0) { + sb.append(" && seg[").append(i).append("].startsWith(\"") + .append(RestClientAnnotationProcessor.escape(prefix)).append("\")"); + } + if (suffix.length() > 0) { + sb.append(" && seg[").append(i).append("].endsWith(\"") + .append(RestClientAnnotationProcessor.escape(suffix)).append("\")"); + } + sb.append(" && seg[").append(i).append("].length() > ") + .append(prefix.length() + suffix.length()); + } + } + return sb.toString(); + } + + private static void emitRoute(StringBuilder sb, Op op) { + String[] template = splitTemplate(op.pathTemplate); + sb.append(" if(").append(routeCondition(op)).append(") {\n"); + // Locals are positional (_a0, _a1, ...) rather than the parameter's own name: + // a @Body parameter called "body" would otherwise shadow dispatch()'s own + // body argument and fail to compile. Generated identifiers must not be able + // to collide with the generator's. + for (int pi = 0; pi < op.params.size(); pi++) { + Param p = op.params.get(pi); + sb.append(" ").append(p.javaType).append(" _a").append(pi).append(" = "); + if ("path".equals(p.bindKind)) { + int idx = placeholderIndex(template, p.bindName); + if (idx < 0) { + sb.append(fromText(p.javaType, "null")); + } else { + // Only the part BETWEEN the literals is the value. The + // condition above has already proved both are present, so the + // arithmetic here cannot go out of range. + String slice = "seg[" + idx + "]"; + int prefixLength = placeholderPrefix(template[idx]).length(); + int suffixLength = placeholderSuffix(template[idx]).length(); + if (prefixLength > 0 || suffixLength > 0) { + slice = slice + ".substring(" + prefixLength + + (suffixLength > 0 ? ", " + slice + ".length() - " + suffixLength : "") + + ")"; + } + sb.append(fromText(p.javaType, "decodePath(" + slice + ")")); + } + } else if ("query".equals(p.bindKind)) { + sb.append(fromText(p.javaType, "queryParam(query, \"" + + RestClientAnnotationProcessor.escape(p.bindName) + "\")")); + } else if ("header".equals(p.bindKind)) { + sb.append(fromText(p.javaType, "header(headers, \"" + + RestClientAnnotationProcessor.escape(p.bindName) + "\")")); + } else if ("cookie".equals(p.bindKind)) { + sb.append(fromText(p.javaType, "cookie(headers, \"" + + RestClientAnnotationProcessor.escape(p.bindName) + "\")")); + } else { + sb.append(fromBody(p.javaType)); + } + sb.append(";\n"); + } + sb.append(" "); + if (!"void".equals(op.returnType)) { + sb.append(op.returnType).append(" _result = "); + } + sb.append("impl.").append(op.name).append('('); + for (int i = 0; i < op.params.size(); i++) { + if (i > 0) sb.append(", "); + sb.append("_a").append(i); + } + sb.append(");\n"); + if ("void".equals(op.returnType)) { + sb.append(" return \"\";\n"); + } else { + sb.append(" return ").append(toJsonValue(op.returnType, "_result")).append(";\n"); + } + sb.append(" }\n"); + } + + /// Wraps a String-valued expression in the conversion its target type needs. + /// A null stays null for the boxed types rather than throwing, so an absent + /// optional query parameter is not a 500. + private static String fromText(String javaType, String expr) { + if ("java.lang.String".equals(javaType)) return expr; + if ("int".equals(javaType)) return "parseInt(" + expr + ")"; + if ("long".equals(javaType)) return "parseLong(" + expr + ")"; + if ("boolean".equals(javaType)) return "parseBool(" + expr + ")"; + if ("double".equals(javaType)) return "parseDouble(" + expr + ")"; + // Parsed AT the target width, not parsed wide and cast down. A cast + // wraps: "40000" for a short became -25536 and "256" for a byte became + // 0, so the handler ran on a number the client never sent, from a value + // the client controls. The boxed forms below were always right about + // this, because Short.valueOf throws -- only the primitives were cast. + if ("float".equals(javaType)) return "parseFloat(" + expr + ")"; + if ("short".equals(javaType)) return "parseShort(" + expr + ")"; + if ("byte".equals(javaType)) return "parseByte(" + expr + ")"; + if ("java.lang.Integer".equals(javaType)) return "boxInt(" + expr + ")"; + if ("java.lang.Long".equals(javaType)) return "boxLong(" + expr + ")"; + if ("java.lang.Double".equals(javaType)) return "boxDouble(" + expr + ")"; + if ("java.lang.Float".equals(javaType)) return "boxFloat(" + expr + ")"; + if ("java.lang.Short".equals(javaType)) return "boxShort(" + expr + ")"; + if ("java.lang.Byte".equals(javaType)) return "boxByte(" + expr + ")"; + if ("java.lang.Boolean".equals(javaType)) return "boxBoolean(" + expr + ")"; + return expr; + } + + /// The request body, converted to the handler's parameter type. + /// + /// The body is whatever the client sent, so its SHAPE is attacker controlled: + /// a route declaring a DTO can be handed a string, a number or an array. None + /// of these conversions may therefore rest on a cast. ParparVM's CHECKCAST is + /// unchecked by default (see CLAUDE.md), so `(Map)body` over a String does not + /// throw -- it reads a String's header as a Map's and the process dies, which + /// on a server takes every in-flight connection with it. Every path below + /// either tests with instanceof or converts through text. + private static String fromBody(String javaType) { + // The STRICT helper: a declared String body must have arrived as a JSON + // string. The lenient one below exists for scalars, where converting + // through text is the point. + if ("java.lang.String".equals(javaType)) return "bodyAsString(body)"; + if (isCollectionShape(javaType)) { + String element = javaType.substring(javaType.indexOf('<') + 1, javaType.length() - 1); + // A Set parameter has to receive a Set. bodyAsList hands back an + // ArrayList, and casting that to Set is exactly the cast the comment + // above warns about: the JVM throws ClassCastException before the + // handler runs, and the translated target does not check at all, so it + // carries an ArrayList in a Set-typed field until something reads it as + // one. setFromList converts instead of asserting. + boolean isSet = javaType.startsWith("java.util.Set<"); + String decoded; + if (element.startsWith("java.")) { + // Converted element by element, not handed over raw. The JSON reader + // produces Long for every integer and Double for every real, so a + // List arrives full of Longs: on the JVM the handler gets a + // ClassCastException the first time it reads one, and on the + // translated target the cast is unchecked, so it reads an Integer's + // fields out of a Long and carries on. The DTO branch below already + // converts; this one used not to. + decoded = "listOfValues(bodyAsList(body), new FromValue() {\n" + + " public Object convert(Object v) { return " + + fieldFromJson(element, "v") + "; }\n" + + " })"; + } else { + decoded = "listFromMaps(bodyAsList(body), new FromMap() {\n" + + " public Object convert(java.util.Map m) { return " + + codecFor(element) + ".fromMap(m); }\n" + + " })"; + } + if (isSet) { + decoded = "setFromList(" + decoded + ")"; + } + return "(" + javaType + ")(Object)" + decoded; + } + // A primitive or boxed scalar goes through the same text conversion the + // query and path parameters use, so a JSON number reaching an `int` body + // behaves the same as one reaching an `int` query parameter. + if (javaType.indexOf('.') < 0 || isBoxedScalar(javaType)) { + return fromText(javaType, "bodyAsText(body)"); + } + if (javaType.startsWith("java.")) { + return guardedCast(javaType, "body"); + } + return codecFor(javaType) + ".fromMap(bodyAsMap(body))"; + } + + private static boolean isBoxedScalar(String javaType) { + return "java.lang.Integer".equals(javaType) || "java.lang.Long".equals(javaType) + || "java.lang.Double".equals(javaType) || "java.lang.Float".equals(javaType) + || "java.lang.Short".equals(javaType) || "java.lang.Byte".equals(javaType) + || "java.lang.Boolean".equals(javaType); + } + + /// `expr` narrowed to `javaType` when it already is one, and null when it is + /// not -- an instanceof rather than a cast, for the reason in {@link #fromBody}. + /// `expr` is evaluated twice, so it must stay side-effect free (it is always a + /// local or a Map read). + private static String guardedCast(String javaType, String expr) { + String raw = javaType; + int generic = raw.indexOf('<'); + if (generic > 0) raw = raw.substring(0, generic); + return "(" + javaType + ")(Object)(" + expr + " instanceof " + raw + + " ? " + expr + " : null)"; + } + + /// The handler's return value, converted to something the JSON writer accepts. + private static String toJsonValue(String javaType, String expr) { + if (isCollectionShape(javaType)) { + String element = javaType.substring(javaType.indexOf('<') + 1, javaType.length() - 1); + if (element.startsWith("java.")) { + // Handed to the writer as it stands, Set included: Json.write emits any + // Collection as an array. Converting a Set to a List here would fix this + // one expression and leave a Set reached through a Map or a DTO field + // still writing itself as a quoted toString(), so the writer is where + // that belongs. + return expr; + } + return "listToMaps(" + expr + ", new ToMap() {\n" + + " public java.util.Map convert(Object o) { return " + + codecFor(element) + ".toMap((" + element + ")o); }\n" + + " })"; + } + if (javaType.startsWith("java.") || javaType.indexOf('.') < 0) { + return expr; + } + return codecFor(javaType) + ".toMap(" + expr + ")"; + } + + private static void emitHelpers(StringBuilder sb) { + sb.append(" /** Converts one element of a decoded JSON array into a DTO. */\n"); + sb.append(" private interface FromMap { Object convert(java.util.Map m); }\n"); + sb.append(" private interface ToMap { java.util.Map convert(Object o); }\n\n"); + emitValueCoercion(sb); + sb.append(" /** Converts one element of a decoded JSON array to its declared type. */\n"); + sb.append(" private interface FromValue { Object convert(Object v); }\n\n"); + sb.append(" private static java.util.List listOfValues(java.util.List raw, FromValue f) {\n"); + sb.append(" if(raw == null) return null;\n"); + sb.append(" java.util.List out = new java.util.ArrayList();\n"); + sb.append(" for(int i = 0 ; i < raw.size() ; i++) {\n"); + sb.append(" out.add(f.convert(raw.get(i)));\n"); + sb.append(" }\n"); + sb.append(" return out;\n"); + sb.append(" }\n\n"); + sb.append(" private static java.util.List listFromMaps(java.util.List raw, FromMap f) {\n"); + sb.append(" if(raw == null) return null;\n"); + sb.append(" java.util.List out = new java.util.ArrayList();\n"); + sb.append(" for(int i = 0 ; i < raw.size() ; i++) {\n"); + sb.append(" Object e = raw.get(i);\n"); + // A non-map element is the CLIENT being wrong, not a null. Substituting + // null for it handed the handler a list with a hole in it -- and the + // handler dereferences the DTO and answers 500, for input that should + // have been a 400. A JSON null stays a null, because that is a value the + // client really sent. + sb.append(" if(e != null && !(e instanceof java.util.Map)) {\n"); + sb.append(" throw new IllegalArgumentException(\"element \" + i" + + " + \" of the body is \" + e.getClass().getName()" + + " + \", not an object\");\n"); + sb.append(" }\n"); + sb.append(" out.add(e == null ? null : f.convert((java.util.Map)e));\n"); + sb.append(" }\n"); + sb.append(" return out;\n"); + sb.append(" }\n\n"); + sb.append(" private static java.util.List listToMaps(java.util.Collection raw, ToMap f) {\n"); + sb.append(" if(raw == null) return null;\n"); + sb.append(" java.util.List out = new java.util.ArrayList();\n"); + sb.append(" java.util.Iterator it = raw.iterator();\n"); + sb.append(" while(it.hasNext()) {\n"); + sb.append(" Object e = it.next();\n"); + sb.append(" out.add(e == null ? null : f.convert(e));\n"); + sb.append(" }\n"); + sb.append(" return out;\n"); + sb.append(" }\n\n"); + sb.append(" /** The body as a JSON object, or a 400 -- never a cast. */\n"); + sb.append(" private static java.util.Map bodyAsMap(Object body) {\n"); + sb.append(" if(body == null || body instanceof java.util.Map) return (java.util.Map)body;\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON object is required in the request body\");\n"); + sb.append(" }\n\n"); + sb.append(" /** The body as a JSON array, or a 400 -- never a cast. */\n"); + sb.append(" private static java.util.List bodyAsList(Object body) {\n"); + sb.append(" if(body == null || body instanceof java.util.List) return (java.util.List)body;\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON array is required in the request body\");\n"); + sb.append(" }\n\n"); + // Declaring @Body String does not make the body a string. A client can + // send the number 1 or an object, and String.valueOf turned those into + // "1" and "{a=1}" as though they had been sent as JSON strings -- the + // same coercion the DTO field path was fixed for, on the top-level body. + sb.append(" private static String bodyAsString(Object body) {\n"); + sb.append(" if(body == null || body instanceof String) { return (String)body; }\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON string is required in the " + + "request body, not \" + body.getClass().getName());\n"); + sb.append(" }\n\n"); + // The lenient twin, and only for scalars: `@Body int` is fed by rendering + // whatever arrived and parsing it, so that a JSON number reaching an int + // body behaves like one reaching an int query parameter. Widening this to + // String is what let an object arrive as "{a=1}". + sb.append(" private static String bodyAsText(Object body) {\n"); + sb.append(" return body == null ? null : String.valueOf(body);\n"); + sb.append(" }\n\n"); + sb.append(" private static String stripQuery(String rawPath) {\n"); + sb.append(" if(rawPath == null) return \"\";\n"); + sb.append(" int q = rawPath.indexOf('?');\n"); + sb.append(" return q < 0 ? rawPath : rawPath.substring(0, q);\n"); + sb.append(" }\n\n"); + sb.append(" private static String queryOf(String rawPath) {\n"); + sb.append(" if(rawPath == null) return \"\";\n"); + sb.append(" int q = rawPath.indexOf('?');\n"); + sb.append(" return q < 0 ? \"\" : rawPath.substring(q + 1);\n"); + sb.append(" }\n\n"); + sb.append(" private static String[] split(String path) {\n"); + sb.append(" return splitOn(path, '/');\n"); + sb.append(" }\n\n"); + sb.append(" private static String queryParam(String query, String name) {\n"); + sb.append(" if(query == null || query.length() == 0) return null;\n"); + sb.append(" String[] pairs = splitOn(query, '&');\n"); + sb.append(" for(int i = 0 ; i < pairs.length ; i++) {\n"); + sb.append(" int eq = pairs[i].indexOf('=');\n"); + // The NAME is decoded before it is compared. A legal annotation name that has + // to be encoded on the wire -- @Query("filter[name]") goes out as + // filter%5Bname%5D, which is what the generated client sends -- otherwise + // never matched the annotation text, and the two halves of one contract + // failed to bind to each other. + sb.append(" if(eq > 0 && decodeQuery(pairs[i].substring(0, eq)).equals(name)) return decodeQuery(pairs[i].substring(eq + 1));\n"); + sb.append(" }\n"); + sb.append(" return null;\n"); + sb.append(" }\n\n"); + sb.append(" /** Header lookup is case-insensitive: HTTP does not guarantee header case. */\n"); + sb.append(" private static String header(java.util.Map headers, String name) {\n"); + sb.append(" if(headers == null) return null;\n"); + sb.append(" Object direct = headers.get(name);\n"); + sb.append(" if(direct != null) return String.valueOf(direct);\n"); + sb.append(" java.util.Iterator it = headers.keySet().iterator();\n"); + sb.append(" while(it.hasNext()) {\n"); + sb.append(" Object k = it.next();\n"); + sb.append(" if(k != null && String.valueOf(k).equalsIgnoreCase(name)) {\n"); + sb.append(" Object v = headers.get(k);\n"); + sb.append(" return v == null ? null : String.valueOf(v);\n"); + sb.append(" }\n"); + sb.append(" }\n"); + sb.append(" return null;\n"); + sb.append(" }\n\n"); + sb.append(" /** Cookies are not a header of their own; they are pairs inside Cookie. */\n"); + sb.append(" private static String cookie(java.util.Map headers, String name) {\n"); + sb.append(" String raw = header(headers, \"Cookie\");\n"); + sb.append(" if(raw == null) return null;\n"); + sb.append(" String[] pairs = splitOn(raw, ';');\n"); + sb.append(" for(int i = 0 ; i < pairs.length ; i++) {\n"); + sb.append(" String pair = pairs[i].trim();\n"); + sb.append(" int eq = pair.indexOf('=');\n"); + sb.append(" if(eq > 0 && pair.substring(0, eq).trim().equals(name)) return decodeCookie(pair.substring(eq + 1));\n"); + sb.append(" }\n"); + sb.append(" return null;\n"); + sb.append(" }\n\n"); + sb.append(" private static String[] splitOn(String value, char sep) {\n"); + sb.append(" java.util.List parts = new java.util.ArrayList();\n"); + sb.append(" int pos = 0;\n"); + sb.append(" while(true) {\n"); + sb.append(" int next = value.indexOf(sep, pos);\n"); + sb.append(" if(next < 0) { parts.add(value.substring(pos)); break; }\n"); + sb.append(" parts.add(value.substring(pos, next));\n"); + sb.append(" pos = next + 1;\n"); + sb.append(" }\n"); + sb.append(" String[] out = new String[parts.size()];\n"); + sb.append(" for(int i = 0 ; i < out.length ; i++) out[i] = (String)parts.get(i);\n"); + sb.append(" return out;\n"); + sb.append(" }\n\n"); + sb.append(" /** Percent-decoding, plus '+' as space in query values. */\n"); + // '+' means a space only in application/x-www-form-urlencoded, which is what a + // query string and a cookie are. In a PATH segment it is an ordinary character, + // so /items/a+b names "a+b" and decoding it to "a b" hands the handler an id the + // client never sent. + sb.append(" /** A path segment. '+' is literal here, per RFC 3986. */\n"); + sb.append(" private static String decodePath(String value) { return decode(value, false); }\n\n"); + sb.append(" /** A query value, which is form-encoded: '+' is a space. */\n"); + sb.append(" private static String decodeQuery(String value) { return decode(value, true); }\n\n"); + sb.append(" /**\n"); + sb.append(" * A cookie value. Cookie syntax has no plus-to-space rule, and '+' is\n"); + sb.append(" * ordinary in the base64 that session tokens are made of, so folding it\n"); + sb.append(" * to a space corrupts the token and the session with it.\n"); + sb.append(" */\n"); + sb.append(" private static String decodeCookie(String value) { return decode(value, false); }\n\n"); + sb.append(" private static String decode(String value, boolean plusIsSpace) {\n"); + sb.append(" if(value == null) return null;\n"); + sb.append(" if(value.indexOf('%') < 0 && !(plusIsSpace && value.indexOf('+') >= 0)) return value;\n"); + // A run of escapes is one UTF-8 sequence, not one character each. Appending + // %C3%A9 as two chars produced "\u00c3\u00a9" where the client sent one + // accented letter, so consecutive escapes are gathered as bytes and decoded + // together. + sb.append(" StringBuilder out = new StringBuilder();\n"); + sb.append(" byte[] pending = new byte[value.length()];\n"); + sb.append(" int pendingLen = 0;\n"); + sb.append(" for(int i = 0 ; i < value.length() ; i++) {\n"); + sb.append(" char c = value.charAt(i);\n"); + // Integer.parseInt(_, 16) accepts a SIGN, so "%+1" decoded as 1 and "%-1" + // as -1 -- two more spellings of a byte the client never wrote. Two hex + // digits, tested as digits. + sb.append(" if(c == '%' && i + 2 < value.length()\n"); + sb.append(" && hex(value.charAt(i + 1)) >= 0 && hex(value.charAt(i + 2)) >= 0) {\n"); + sb.append(" pending[pendingLen++] =\n"); + sb.append(" (byte)((hex(value.charAt(i + 1)) << 4) | hex(value.charAt(i + 2)));\n"); + sb.append(" i += 2;\n"); + sb.append(" continue;\n"); + sb.append(" }\n"); + sb.append(" if(pendingLen > 0) {\n"); + sb.append(" out.append(decodeUtf8(pending, pendingLen));\n"); + sb.append(" pendingLen = 0;\n"); + sb.append(" }\n"); + sb.append(" if(plusIsSpace && c == '+') { out.append(' '); continue; }\n"); + sb.append(" out.append(c);\n"); + sb.append(" }\n"); + sb.append(" if(pendingLen > 0) out.append(decodeUtf8(pending, pendingLen));\n"); + sb.append(" return out.toString();\n"); + sb.append(" }\n\n"); + sb.append(" /** The gathered escape bytes as text. Malformed input keeps its bytes rather than throwing. */\n"); + sb.append(" private static int hex(char c) {\n"); + sb.append(" if(c >= '0' && c <= '9') { return c - '0'; }\n"); + sb.append(" if(c >= 'a' && c <= 'f') { return c - 'a' + 10; }\n"); + sb.append(" if(c >= 'A' && c <= 'F') { return c - 'A' + 10; }\n"); + sb.append(" return -1;\n"); + sb.append(" }\n\n"); + sb.append(" /** Every % must introduce two hex digits; see the router's copy. */\n"); + sb.append(" private static boolean wellFormedEscapes(String value) {\n"); + sb.append(" if(value == null) { return true; }\n"); + sb.append(" for(int i = 0 ; i < value.length() ; i++) {\n"); + sb.append(" if(value.charAt(i) != '%') { continue; }\n"); + sb.append(" if(i + 2 >= value.length()) { return false; }\n"); + sb.append(" if(hex(value.charAt(i + 1)) < 0 || hex(value.charAt(i + 2)) < 0) {\n"); + sb.append(" return false;\n"); + sb.append(" }\n"); + sb.append(" i += 2;\n"); + sb.append(" }\n"); + sb.append(" return true;\n"); + sb.append(" }\n\n"); + sb.append(" private static String decodeUtf8(byte[] bytes, int length) {\n"); + sb.append(" try {\n"); + sb.append(" return new String(bytes, 0, length, \"UTF-8\");\n"); + sb.append(" } catch (java.io.UnsupportedEncodingException err) {\n"); + sb.append(" return new String(bytes, 0, length);\n"); + sb.append(" }\n"); + sb.append(" }\n\n"); + sb.append(" // A missing text value binds to 0 / null rather than throwing: an absent\n"); + sb.append(" // optional query parameter is not a server error.\n"); + sb.append(" private static int parseInt(String v) { return v == null || v.length() == 0 ? 0 : Integer.parseInt(v.trim()); }\n"); + sb.append(" private static long parseLong(String v) { return v == null || v.length() == 0 ? 0L : Long.parseLong(v.trim()); }\n"); + // Double.parseDouble does not FAIL on a value too large for a double: + // 1e999 comes back as infinity, so the handler ran on a number the client + // never sent, and echoing it through Json writes null -- the client gets + // back neither its value nor an error. The SPELLING decides whether an + // infinity was meant, because the parsed value cannot tell 1e999 from + // Infinity. Same test the controller processor's guards use. + sb.append(" private static double parseDouble(String v) {\n"); + sb.append(" if (v == null || v.length() == 0) { return 0d; }\n"); + sb.append(" String t = v.trim();\n"); + sb.append(" double d = Double.parseDouble(t);\n"); + sb.append(" if (Double.isInfinite(d) && t.indexOf(\"Infinity\") < 0) {\n"); + sb.append(" throw new NumberFormatException(\"out of range for double: \" + v);\n"); + sb.append(" }\n"); + sb.append(" return d;\n"); + sb.append(" }\n"); + sb.append(" private static short parseShort(String v) { return v == null || v.length() == 0 ? (short)0 : Short.parseShort(v.trim()); }\n"); + sb.append(" private static byte parseByte(String v) { return v == null || v.length() == 0 ? (byte)0 : Byte.parseByte(v.trim()); }\n"); + // A double outside float range becomes INFINITY on the cast rather than + // failing, so 1e100 reached the handler as an infinite amount. Rejected + // the same way an unparseable number is, which the dispatcher already + // answers 400 for. + sb.append(" private static float parseFloat(String v) {\n"); + sb.append(" if (v == null || v.length() == 0) { return 0f; }\n"); + sb.append(" String t = v.trim();\n"); + sb.append(" double d = Double.parseDouble(t);\n"); + sb.append(" float f = (float)d;\n"); + // Was !Double.isInfinite(d), which is the wrong question: parseDouble + // ("1e999") is ITSELF infinite, so every double-overflowing value read as + // a deliberate infinity and went through. The text is what says it was + // meant. + sb.append(" if (Float.isInfinite(f) && t.indexOf(\"Infinity\") < 0) {\n"); + sb.append(" throw new NumberFormatException(\"out of range for float: \" + v);\n"); + sb.append(" }\n"); + sb.append(" return f;\n"); + sb.append(" }\n"); + sb.append(" private static Integer boxInt(String v) { return v == null || v.length() == 0 ? null : Integer.valueOf(v.trim()); }\n"); + sb.append(" private static Long boxLong(String v) { return v == null || v.length() == 0 ? null : Long.valueOf(v.trim()); }\n"); + // Through the guarded parsers, not valueOf: a boxed binding is the same + // binding with a null for "absent", and it had no overflow check at all. + sb.append(" private static Double boxDouble(String v) { return v == null || v.length() == 0 ? null : Double.valueOf(parseDouble(v)); }\n"); + sb.append(" private static Float boxFloat(String v) { return v == null || v.length() == 0 ? null : Float.valueOf(parseFloat(v)); }\n"); + sb.append(" private static Short boxShort(String v) { return v == null || v.length() == 0 ? null : Short.valueOf(v.trim()); }\n"); + sb.append(" private static Byte boxByte(String v) { return v == null || v.length() == 0 ? null : Byte.valueOf(v.trim()); }\n"); + // NOT Boolean.parseBoolean, which answers false for everything that is not + // "true": "?enabled=treu" reached the handler as an explicit false and the + // client was told nothing, while the same typo in a numeric binding throws + // and comes back as a 400. A present value is either boolean or it is a + // mistake worth reporting. + sb.append(" private static boolean parseBool(String v) {\n"); + sb.append(" if (v == null || v.length() == 0) { return false; }\n"); + sb.append(" String t = v.trim();\n"); + sb.append(" if (t.equalsIgnoreCase(\"true\")) { return true; }\n"); + sb.append(" if (t.equalsIgnoreCase(\"false\")) { return false; }\n"); + sb.append(" throw new IllegalArgumentException(\"not a boolean: \" + v);\n"); + sb.append(" }\n"); + sb.append(" private static Boolean boxBoolean(String v) {\n"); + sb.append(" return v == null || v.length() == 0 ? null : Boolean.valueOf(parseBool(v));\n"); + sb.append(" }\n"); + } + + // ---------------------------------------------------------------- + // DTO codecs + // ---------------------------------------------------------------- + + /// Emits a Map<->DTO codec from the type's public instance fields. Field-based + /// rather than reflective on purpose: ParparVM has no usable reflection and + /// Codename One obfuscates, so a name lookup at runtime would fail in exactly + /// the builds that matter. + private String generateDtoCodec(String binaryName, AnnotatedClass cls, + ProcessorContext ctx) { + String pkg = RestClientAnnotationProcessor.packageOf(binaryName); + String simple = RestClientAnnotationProcessor.simpleName(binaryName); + StringBuilder sb = new StringBuilder(4096); + if (pkg.length() > 0) sb.append("package ").append(pkg).append(";\n\n"); + sb.append("// Auto-generated by cn1:process-annotations for ").append(binaryName).append(". Do not edit.\n"); + sb.append("@SuppressWarnings({\"all\"})\n"); + sb.append("@com.codename1.backend.annotations.Generated\n"); + sb.append("public final class ").append(simple).append("Json {\n"); + sb.append(" private ").append(simple).append("Json() { }\n\n"); + + sb.append(" public static java.util.Map toMap(").append(binaryName).append(" o) {\n"); + sb.append(" if(o == null) return null;\n"); + sb.append(" java.util.Map m = new java.util.LinkedHashMap();\n"); + for (FieldInfo f : transferredFields(cls, ctx)) { + String type = fieldJavaType(f); + sb.append(" m.put(\"").append(RestClientAnnotationProcessor.escape(f.getName())) + .append("\", ").append(fieldToJson(type, "o." + f.getName())).append(");\n"); + } + sb.append(" return m;\n"); + sb.append(" }\n\n"); + + sb.append(" public static ").append(binaryName).append(" fromMap(java.util.Map m) {\n"); + sb.append(" if(m == null) return null;\n"); + sb.append(" ").append(binaryName).append(" o = new ").append(binaryName).append("();\n"); + for (FieldInfo f : transferredFields(cls, ctx)) { + if ((f.getAccess() & org.objectweb.asm.Opcodes.ACC_SYNTHETIC) != 0) continue; + if (f.isFinal()) continue; // cannot be assigned after construction + String type = fieldJavaType(f); + sb.append(" o.").append(f.getName()).append(" = ") + .append(fieldFromJson(type, "m.get(\"" + RestClientAnnotationProcessor.escape(f.getName()) + "\")")) + .append(";\n"); + } + sb.append(" return o;\n"); + sb.append(" }\n\n"); + emitCodecHelpers(sb); + sb.append("}\n"); + return sb.toString(); + } + + /// NOTE: a direct-to-bytes writer was tried here and REVERTED. Emitting + /// `toJson(T, com.codename1.backend.ByteSink)` beside `toMap` is worth a + /// measured +29% on a JSON route (74% -> 94% of Go net/http), because it drops + /// the per-request LinkedHashMap, the key hashing and the instanceof dispatch + /// that walking a map costs. + /// + /// It cannot go here as things stand: generated sources are compiled against + /// `ctx.getOutputClassDir()` and NOTHING ELSE (see the compile call above), so + /// they cannot name a backend type. It happened to work for a backend app, + /// whose own build puts com.codename1.backend in that same directory, and + /// failed for every other project -- including this processor's own tests. + /// + /// To take the 29%, the generated code first needs a type it is allowed to + /// name: either the project's compile classpath reaches the codec compile, or + /// the sink interface lives somewhere generated code may always depend on. + /// That is a deliberate decision about this processor's dependency contract, + /// not a detail to slip in behind a performance patch. + private static String fieldToJson(String type, String expr) { + if (isCollectionShape(type)) { + String element = type.substring(type.indexOf('<') + 1, type.length() - 1); + if (element.startsWith("java.")) return "toValueList(" + expr + ")"; + // A nested DTO list has to become a list of MAPS; handing the writer + // the DTOs themselves serialises them as toString(). + return "toMapList(" + expr + ", new ToMapFn() {\n" + + " public java.util.Map convert(Object o) { return " + + codecFor(element) + ".toMap((" + element + ")o); }\n" + + " })"; + } + if (type.startsWith("java.") || type.indexOf('.') < 0) return expr; + return codecFor(type) + ".toMap(" + expr + ")"; + } + + private static String fieldFromJson(String type, String expr) { + if (isCollectionShape(type)) { + String element = type.substring(type.indexOf('<') + 1, type.length() - 1); + // Both branches below produce a List, so a Set-typed field has to be + // converted rather than cast -- the same fix the request-body path + // needed. Review named the java.* branch; the DTO branch had it too, + // which is why this converts in one place at the end instead. + boolean isSet = type.startsWith("java.util.Set<"); + if (element.startsWith("java.")) { + // Each element converted, exactly as the DTO branch below does. The + // parser produces Long for every integer, so a List FIELD + // arrived full of Longs -- the same defect the collection body had, + // one level further in, and the same silent misread on a target whose + // CHECKCAST does not check. + String decoded = "fromValueList(" + expr + ", new FromValueFn() {\n" + + " public Object convert(Object v) { return " + + fieldFromJson(element, "v") + "; }\n" + + " })"; + if (isSet) { + decoded = "setFromList(" + decoded + ")"; + } + return "(" + type + ")(Object)" + decoded; + } + // Each element is converted through the element codec. Returning the + // decoded Maps as-is -- which this used to do -- gives the handler a + // List whose elements are Maps typed as DTOs: a lie the JVM catches at + // the first field read and ParparVM does not catch at all. + String decodedDtos = "fromMapList(" + expr + ", new FromMapFn() {\n" + + " public Object convert(java.util.Map m) { return " + + codecFor(element) + ".fromMap(m); }\n" + + " })"; + if (isSet) { + decodedDtos = "setFromList(" + decodedDtos + ")"; + } + return "(" + type + ")(Object)" + decodedDtos; + } + if ("java.lang.String".equals(type)) return "asString(" + expr + ")"; + if ("int".equals(type)) return "asInt(" + expr + ")"; + if ("long".equals(type)) return "asLong(" + expr + ")"; + if ("double".equals(type)) return "asDouble(" + expr + ")"; + if ("float".equals(type)) return "asFloat(" + expr + ")"; + if ("short".equals(type)) return "asShort(" + expr + ")"; + if ("byte".equals(type)) return "asByte(" + expr + ")"; + if ("boolean".equals(type)) return "asBoolean(" + expr + ")"; + if ("java.lang.Integer".equals(type)) return "asBoxedInt(" + expr + ")"; + if ("java.lang.Long".equals(type)) return "asBoxedLong(" + expr + ")"; + if ("java.lang.Double".equals(type)) return "asBoxedDouble(" + expr + ")"; + if ("java.lang.Boolean".equals(type)) return "asBoxedBoolean(" + expr + ")"; + // Float, Short and Byte need the same treatment as the three above. The JSON + // reader only ever produces Long or Double, so leaving them to guardedCast + // meant an instanceof against the declared wrapper that never matched, and + // the field silently arrived null with the client's value discarded. + if ("java.lang.Float".equals(type)) return "asBoxedFloat(" + expr + ")"; + if ("java.lang.Short".equals(type)) return "asBoxedShort(" + expr + ")"; + if ("java.lang.Byte".equals(type)) return "asBoxedByte(" + expr + ")"; + // Anything else out of java.* is narrowed with instanceof rather than cast: + // the value came from the wire, so its type is the client's choice. + if (type.startsWith("java.")) return guardedCast(type, expr); + // requireMap, not asMap: asMap answers null for anything that is not one, + // so {"child":1} left the field null and the handler ran on input the + // client did not send -- indistinguishable from an explicit JSON null. + return codecFor(type) + ".fromMap(requireMap(" + expr + "))"; + } + + /** + * The value coercions both generated classes need. + * + * Emitted into the dispatcher as well as the codec because the dispatcher + * converts collection elements too: a List body reaches it as a list of + * Longs, and the conversion that fixes that is written in terms of these. They + * were in the codec alone, so the dispatcher referred to helpers it did not have + * and simply failed to compile. + */ + private static void emitValueCoercion(StringBuilder sb) { + sb.append(" // The JSON reader produces Long for integers and Double for reals, so every\n"); + sb.append(" // numeric read goes through Number rather than casting to the field's type.\n"); + // String.valueOf turns ANYTHING into a string, so a number arrived as + // "1" and a whole object as "{x=1}" -- values the declared JSON shape + // never allowed, handed to the handler as though the client had sent + // them. A JSON string is a string; anything else is the client being + // wrong, and null is still null. + sb.append(" private static String asString(Object v) {\n"); + sb.append(" if (v == null || v instanceof String) { return (String)v; }\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON string is required, not \"" + + " + v.getClass().getName());\n"); + sb.append(" }\n"); + // Range-checked, not narrowed. The parser answers a Long for any JSON + // integer, and intValue() on 2147483648 is -2147483648 -- so an id, a count + // or an amount reached the handler as a DIFFERENT number from the one the + // client sent, with nothing raised. A value that does not fit is the + // client's mistake and is reported as one. + // Whole numbers only. The parser answers a Double for any JSON real, and + // longValue() on 1.9 is 1 -- so a fractional id, count or amount reached + // the handler as a DIFFERENT number from the one the client sent, and the + // range check below never sees it because 1 is perfectly in range. A value + // that is not integral is the client's mistake and is reported as one. + sb.append(" private static long integral(Object v, String type) {\n"); + sb.append(" double d = ((Number)v).doubleValue();\n"); + sb.append(" if (Double.isNaN(d) || Double.isInfinite(d) || d != Math.floor(d)) {\n"); + sb.append(" throw new IllegalArgumentException(\"not a whole number for \" + type + \": \" + v);\n"); + sb.append(" }\n"); + // Range too, and BEFORE the narrowing rather than after it. longValue() + // SATURATES: 1e20 comes back as Long.MAX_VALUE instead of throwing, so an + // id or an amount too large to represent arrived as a plausible number + // that is not the one the client sent. Only a Double can be out of range + // here -- a Long already is one -- so the bound is tested on the double. + sb.append(" if (!(v instanceof Long) && (d < -9.223372036854776E18 || d >= 9.223372036854776E18)) {\n"); + sb.append(" throw new IllegalArgumentException(\"out of range for \" + type + \": \" + v);\n"); + sb.append(" }\n"); + sb.append(" return ((Number)v).longValue();\n"); + sb.append(" }\n"); + sb.append(" private static int asInt(Object v) {\n"); + sb.append(" if (v instanceof Number) {\n"); + sb.append(" long asLong = integral(v, \"int\");\n"); + sb.append(" if (asLong < Integer.MIN_VALUE || asLong > Integer.MAX_VALUE) {\n"); + sb.append(" throw new IllegalArgumentException(\"out of range for int: \" + v);\n"); + sb.append(" }\n"); + sb.append(" return (int)asLong;\n"); + sb.append(" }\n"); + // A JSON STRING is not a number. These helpers decode a value the parser + // has already typed, so "123" where the contract declares an int is the + // client disagreeing with the contract -- and parsing it anyway means the + // handler cannot tell the two apart, while the equivalent request to the + // generated CLIENT could never have produced it. The text bindings are a + // different path on purpose: a query parameter really does arrive as text, + // and fromText/parseInt still parse it. + sb.append(" if (v != null) {\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON number is required, not \"\n"); + sb.append(" + v.getClass().getName() + \": \" + v);\n"); + sb.append(" }\n"); + sb.append(" return 0;\n"); + sb.append(" }\n"); + sb.append(" private static short asShort(Object v) {\n"); + sb.append(" int narrowed = asInt(v);\n"); + sb.append(" if (narrowed < Short.MIN_VALUE || narrowed > Short.MAX_VALUE) {\n"); + sb.append(" throw new IllegalArgumentException(\"out of range for short: \" + v);\n"); + sb.append(" }\n"); + sb.append(" return (short)narrowed;\n"); + sb.append(" }\n"); + sb.append(" private static byte asByte(Object v) {\n"); + sb.append(" int narrowed = asInt(v);\n"); + sb.append(" if (narrowed < Byte.MIN_VALUE || narrowed > Byte.MAX_VALUE) {\n"); + sb.append(" throw new IllegalArgumentException(\"out of range for byte: \" + v);\n"); + sb.append(" }\n"); + sb.append(" return (byte)narrowed;\n"); + sb.append(" }\n"); + // Same rule as asInt above, and for the same reason. + sb.append(" private static long asLong(Object v) {\n"); + sb.append(" if (v instanceof Number) { return integral(v, \"long\"); }\n"); + sb.append(" if (v != null) {\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON number is required, not \"\n"); + sb.append(" + v.getClass().getName() + \": \" + v);\n"); + sb.append(" }\n"); + sb.append(" return 0L;\n"); + sb.append(" }\n"); + sb.append(" private static double asDouble(Object v) {\n"); + sb.append(" if (v instanceof Number) { return ((Number)v).doubleValue(); }\n"); + sb.append(" if (v != null) {\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON number is required, not \"\n"); + sb.append(" + v.getClass().getName() + \": \" + v);\n"); + sb.append(" }\n"); + sb.append(" return 0d;\n"); + sb.append(" }\n"); + // A cast to float SATURATES: a perfectly ordinary finite 1e100 becomes + // infinity, which is not a number JSON can express and is not the one the + // client sent. The scalar text path refuses it; a DTO field has to as + // well, or the same value is accepted or rejected by where it appears. + sb.append(" private static float asFloat(Object v) {\n"); + sb.append(" double d = asDouble(v);\n"); + sb.append(" float f = (float)d;\n"); + sb.append(" if (Float.isInfinite(f) && !Double.isInfinite(d)) {\n"); + sb.append(" throw new IllegalArgumentException(\"out of range for float: \" + v);\n"); + sb.append(" }\n"); + sb.append(" return f;\n"); + sb.append(" }\n"); + // Boolean.parseBoolean answers FALSE for everything that is not "true", + // so {"good":1} and {"good":"invalid"} both reached the handler as an + // explicit false the client never sent. This is a JSON body, where the + // value has a real type -- unlike the text bindings, where several + // spellings are a deliberate convention -- so anything that is not a + // boolean is the client being wrong and is answered 400. + sb.append(" private static boolean asBoolean(Object v) {\n"); + sb.append(" if (v instanceof Boolean) { return ((Boolean)v).booleanValue(); }\n"); + sb.append(" if (v == null) { return false; }\n"); + sb.append(" throw new IllegalArgumentException(\"not a boolean: \" + v);\n"); + sb.append(" }\n"); + sb.append(" private static Integer asBoxedInt(Object v) { return v == null ? null : Integer.valueOf(asInt(v)); }\n"); + sb.append(" private static Long asBoxedLong(Object v) { return v == null ? null : Long.valueOf(asLong(v)); }\n"); + sb.append(" private static Double asBoxedDouble(Object v) { return v == null ? null : Double.valueOf(asDouble(v)); }\n"); + sb.append(" private static Boolean asBoxedBoolean(Object v) { return v == null ? null : Boolean.valueOf(asBoolean(v)); }\n"); + sb.append(" private static Float asBoxedFloat(Object v) { return v == null ? null : Float.valueOf(asFloat(v)); }\n"); + sb.append(" private static Short asBoxedShort(Object v) { return v == null ? null : Short.valueOf(asShort(v)); }\n"); + sb.append(" private static Byte asBoxedByte(Object v) { return v == null ? null : Byte.valueOf(asByte(v)); }\n"); + sb.append(" /** A decoded value narrowed to a JSON object, or null -- never a cast. */\n"); + sb.append(" private static java.util.Map asMap(Object v) { return v instanceof java.util.Map ? (java.util.Map)v : null; }\n"); + // The difference between "the client sent null" and "the client sent + // something that is not an object". The first is a value; the second is + // a mistake, and answering 400 for it is the whole point of decoding. + sb.append(" private static java.util.Map requireMap(Object v) {\n"); + sb.append(" if (v == null) { return null; }\n"); + sb.append(" if (!(v instanceof java.util.Map)) {\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON object is required, not \"" + + " + v.getClass().getName());\n"); + sb.append(" }\n"); + sb.append(" return (java.util.Map)v;\n"); + sb.append(" }\n"); + sb.append(" private static java.util.List asList(Object v) { return v instanceof java.util.List ? (java.util.List)v : null; }\n"); + sb.append(" /** A decoded array as a Set, preserving the order it arrived in. */\n"); + sb.append(" private static java.util.Set setFromList(java.util.List v) {\n"); + sb.append(" return v == null ? null : new java.util.LinkedHashSet(v);\n"); + sb.append(" }\n"); + } + + private static void emitCodecHelpers(StringBuilder sb) { + emitValueCoercion(sb); + sb.append(" private interface ToMapFn { java.util.Map convert(Object o); }\n"); + sb.append(" private interface FromMapFn { Object convert(java.util.Map m); }\n"); + sb.append(" private interface FromValueFn { Object convert(Object v); }\n"); + sb.append(" /** Converts each element of a decoded array to the field's element type. */\n"); + sb.append(" private static java.util.List fromValueList(Object raw, FromValueFn f) {\n"); + // asList answers null for anything that is not one, so an object or a + // scalar where an array was declared left the field null -- the client's + // mistake made indistinguishable from an explicit JSON null. + sb.append(" if(raw == null) return null;\n"); + sb.append(" if(!(raw instanceof java.util.List)) {\n"); + sb.append(" throw new IllegalArgumentException(\"a JSON array is required, not \"" + + " + raw.getClass().getName());\n"); + sb.append(" }\n"); + sb.append(" java.util.List in = (java.util.List)raw;\n"); + sb.append(" java.util.List out = new java.util.ArrayList();\n"); + sb.append(" for(int i = 0 ; i < in.size() ; i++) {\n"); + sb.append(" out.add(f.convert(in.get(i)));\n"); + sb.append(" }\n"); + sb.append(" return out;\n"); + sb.append(" }\n"); + sb.append(" private static java.util.List toValueList(java.util.Collection raw) {\n"); + sb.append(" if(raw == null) return null;\n"); + sb.append(" java.util.List out = new java.util.ArrayList();\n"); + sb.append(" java.util.Iterator it = raw.iterator();\n"); + sb.append(" while(it.hasNext()) out.add(it.next());\n"); + sb.append(" return out;\n"); + sb.append(" }\n"); + sb.append(" private static java.util.List toMapList(java.util.Collection raw, ToMapFn f) {\n"); + sb.append(" if(raw == null) return null;\n"); + sb.append(" java.util.List out = new java.util.ArrayList();\n"); + sb.append(" java.util.Iterator it = raw.iterator();\n"); + sb.append(" while(it.hasNext()) { Object e = it.next(); out.add(e == null ? null : f.convert(e)); }\n"); + sb.append(" return out;\n"); + sb.append(" }\n"); + sb.append(" private static java.util.List fromMapList(Object raw, FromMapFn f) {\n"); + sb.append(" if(!(raw instanceof java.util.List)) return null;\n"); + sb.append(" java.util.List src = (java.util.List)raw;\n"); + sb.append(" java.util.List out = new java.util.ArrayList();\n"); + sb.append(" for(int i = 0 ; i < src.size() ; i++) {\n"); + sb.append(" Object e = src.get(i);\n"); + // The same rule listFromMaps takes: a non-null element that is not an + // object is the client being wrong, and substituting null for it hands + // the handler a collection with a hole where a DTO should be. + sb.append(" if(e != null && !(e instanceof java.util.Map)) {\n"); + sb.append(" throw new IllegalArgumentException(\"element \" + i" + + " + \" is \" + e.getClass().getName() + \", not an object\");\n"); + sb.append(" }\n"); + sb.append(" out.add(e == null ? null : f.convert((java.util.Map)e));\n"); + sb.append(" }\n"); + sb.append(" return out;\n"); + sb.append(" }\n"); + } + + private static String[] splitTemplate(String template) { + String t = template == null ? "" : template; + // ORIGIN-FORM first. The runtime splits the incoming path with the same + // algorithm, so "/notes" arrives as ["", "notes"] while a contract written + // as @GET("notes") -- which the client resolves against a base URL ending + // in "/" and requests as /notes -- split to ["notes"] and could never match + // on length. The empty template had the same problem against "/". + if (t.length() == 0 || t.charAt(0) != '/') { + t = "/" + t; + } + List parts = new ArrayList(); + int pos = 0; + while (pos <= t.length()) { + int next = t.indexOf('/', pos); + if (next < 0) { parts.add(t.substring(pos)); break; } + parts.add(t.substring(pos, next)); + pos = next + 1; + } + return parts.toArray(new String[parts.size()]); + } + + /** + * Whether this segment carries a placeholder at all -- alone or with literal + * text around it. + * + * The CLIENT generator has always substituted {name} anywhere in the template, + * so /files/{name}.json produced a working client while this half saw no + * placeholder, reported that the @Path was unbound, and refused the contract. + * One annotation cannot mean two things in the two halves generated from it. + */ + private static boolean isPlaceholder(String segment) { + int open = segment.indexOf('{'); + return open >= 0 && segment.indexOf('}', open + 1) > open + 1; + } + + /** The name inside this segment's placeholder, or null when it has none. */ + private static String placeholderName(String segment) { + int open = segment.indexOf('{'); + if (open < 0) { + return null; + } + int close = segment.indexOf('}', open + 1); + return close > open + 1 ? segment.substring(open + 1, close) : null; + } + + /** The literal text before this segment's placeholder. */ + private static String placeholderPrefix(String segment) { + int open = segment.indexOf('{'); + return open < 0 ? "" : segment.substring(0, open); + } + + /** The literal text after it. */ + private static String placeholderSuffix(String segment) { + int open = segment.indexOf('{'); + if (open < 0) { + return ""; + } + int close = segment.indexOf('}', open + 1); + return close < 0 ? "" : segment.substring(close + 1); + } + + /** + * Whether this segment holds more than one placeholder. + * + * Refused rather than matched: "{a}-{b}" has no single reading -- the split + * point between the two values is a guess -- and guessing it here would make + * the server bind something the client never meant. Named explicitly so the + * developer is told, instead of the shape silently not matching. + */ + private static boolean hasSecondPlaceholder(String segment) { + int open = segment.indexOf('{'); + if (open < 0) { + return false; + } + int close = segment.indexOf('}', open + 1); + return close >= 0 && segment.indexOf('{', close + 1) >= 0; + } + + private static int placeholderIndex(String[] template, String name) { + for (int i = 0; i < template.length; i++) { + if (name.equals(placeholderName(template[i]))) { + return i; + } + } + return -1; + } +} diff --git a/maven/codenameone-maven-plugin/src/main/resources/META-INF/services/com.codename1.maven.annotations.AnnotationProcessor b/maven/codenameone-maven-plugin/src/main/resources/META-INF/services/com.codename1.maven.annotations.AnnotationProcessor index 965c0ff6c46..b0bf6975492 100644 --- a/maven/codenameone-maven-plugin/src/main/resources/META-INF/services/com.codename1.maven.annotations.AnnotationProcessor +++ b/maven/codenameone-maven-plugin/src/main/resources/META-INF/services/com.codename1.maven.annotations.AnnotationProcessor @@ -3,8 +3,10 @@ com.codename1.maven.processors.MappingAnnotationProcessor com.codename1.maven.processors.BindingAnnotationProcessor com.codename1.maven.processors.OrmAnnotationProcessor com.codename1.maven.processors.RestClientAnnotationProcessor +com.codename1.maven.processors.RestServerAnnotationProcessor com.codename1.maven.processors.ProtoMessageAnnotationProcessor com.codename1.maven.processors.GrpcClientAnnotationProcessor com.codename1.maven.processors.GraphQLClientAnnotationProcessor com.codename1.maven.processors.AppIntentAnnotationProcessor com.codename1.maven.processors.BuildHintAnnotationProcessor +com.codename1.maven.processors.RestControllerAnnotationProcessor diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java new file mode 100644 index 00000000000..d06fa5d6740 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestControllerAnnotationProcessorTest.java @@ -0,0 +1,1432 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven.processors; + +import com.codename1.maven.annotations.AnnotatedClass; +import com.codename1.maven.annotations.ClassScanner; +import com.codename1.maven.annotations.JavaSourceCompiler; +import com.codename1.maven.annotations.ProcessorContext; +import com.codename1.backend.HttpServer; +import com.codename1.backend.Json; +import org.apache.maven.plugin.logging.SystemStreamLog; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/// Proves the generated router ROUTES. The class it produces is compiled, loaded +/// and called with real `HttpServer.Request` objects here, because a router that +/// compiles and matches nothing is exactly the failure this is for -- and because +/// the matching runs on the request's bytes, which only a real Request has. +public class RestControllerAnnotationProcessorTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private static final String CONTROLLER_SOURCE = + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import java.util.*;\n" + + "@RestController\n" + + "@RequestMapping(\"/api\")\n" + + "public class Notes {\n" + + " @GetMapping(\"/healthz\")\n" + + " public String health() { return \"ok\"; }\n" + + " @GetMapping(\"/notes/{id}\")\n" + + " public Map note(@PathVariable(\"id\") String id) {\n" + + " Map m = new LinkedHashMap(); m.put(\"id\", id); return m;\n" + + " }\n" + + " @GetMapping(\"/notes/{id}/tags/{tag}\")\n" + + " public Map tag(@PathVariable(\"id\") String id, @PathVariable(\"tag\") String tag) {\n" + + " Map m = new LinkedHashMap(); m.put(\"id\", id); m.put(\"tag\", tag); return m;\n" + + " }\n" + + " @GetMapping(\"/search\")\n" + + " public Map search(@RequestParam(\"q\") String q,\n" + + " @RequestParam(value=\"page\", defaultValue=\"7\") int page) {\n" + + " Map m = new LinkedHashMap(); m.put(\"q\", q);\n" + + " m.put(\"page\", Integer.valueOf(page)); return m;\n" + + " }\n" + + " @GetMapping(\"/agent\")\n" + + " public String agent(@RequestHeader(\"user-agent\") String ua) { return ua; }\n" + + " @PostMapping(\"/notes\")\n" + + " @ResponseStatus(201)\n" + + " public Map create(@RequestBody Map body) { return body; }\n" + + " @GetMapping(\"/tags\")\n" + + " public Set tags() { return new LinkedHashSet(Arrays.asList(\"a\", \"b\")); }\n" + + " @GetMapping(\"/boom\")\n" + + " public String boom() throws java.io.IOException {\n" + + " throw new java.io.IOException(\"from the handler\");\n" + + " }\n" + + "}\n"; + + @Test + public void routesEveryBindingKind() throws Exception { + Router router = generate(CONTROLLER_SOURCE); + + assertEquals("ok", router.text("GET", "/api/healthz")); + // A query string is not part of the route. Matching the whole target instead + // of the path is the bug this asserts against. + assertEquals("ok", router.text("GET", "/api/healthz?probe=1")); + assertEquals("{\"id\":\"42\"}", router.text("GET", "/api/notes/42")); + assertEquals("{\"id\":\"42\"}", router.text("GET", "/api/notes/42?x=1")); + assertEquals("{\"id\":\"a b\"}", router.text("GET", "/api/notes/a%20b")); + // '+' is a literal in a path segment; it means a space only in a query. + assertEquals("{\"id\":\"a+b\"}", router.text("GET", "/api/notes/a+b")); + assertEquals("{\"id\":\"42\",\"tag\":\"red\"}", + router.text("GET", "/api/notes/42/tags/red")); + assertEquals("{\"q\":\"hi\",\"page\":7}", router.text("GET", "/api/search?q=hi")); + assertEquals("{\"q\":\"hi\",\"page\":3}", router.text("GET", "/api/search?q=hi&page=3")); + // Not the default: defaultValue is documented as "used when the request + // omits it", and "zz" is not an omission. Binding it to 7 handed the + // handler a page the client never asked for, and neither could tell. + Object malformed = router.call("GET", "/api/search?q=hi&page=zz", null); + assertNotNull(malformed); + assertEquals(400, Router.statusOf(malformed)); + assertEquals("[\"a\",\"b\"]", router.text("GET", "/api/tags")); + } + + @Test + public void bindsBodyAndStatus() throws Exception { + Router router = generate(CONTROLLER_SOURCE); + Object response = router.call("POST", "/api/notes", "{\"body\":\"hi\"}"); + assertNotNull("POST /api/notes did not match", response); + assertEquals(201, ((HttpServer.Response) response).getStatus()); + assertEquals("{\"body\":\"hi\"}", Router.bodyOf(response)); + } + + @Test + public void doesNotMatchWhatItShouldNot() throws Exception { + Router router = generate(CONTROLLER_SOURCE); + // A path variable is ONE segment: without that guard /notes/{id} swallows + // /notes/1/2 and hands the method "1/2" as the id. + assertNull(router.call("GET", "/api/notes/1/2", null)); + assertNull(router.call("GET", "/api/nope", null)); + assertNull("the method is part of the route", router.call("POST", "/api/healthz", null)); + assertNull("the class-level base path applies", router.call("GET", "/healthz", null)); + } + + @Test + public void aHandlerMayThrow() throws Exception { + Router router = generate(CONTROLLER_SOURCE); + try { + router.call("GET", "/api/boom", null); + fail("the handler's IOException should reach the server"); + } catch (java.lang.reflect.InvocationTargetException err) { + assertTrue(err.getCause() instanceof java.io.IOException); + } + } + + @Test + public void namesTheBootstrapForThePackagingGoal() throws Exception { + ProcessorContext ctx = run(compile(CONTROLLER_SOURCE)); + byte[] name = ctx.getEmittedResources() + .get(RestControllerAnnotationProcessor.MAIN_CLASS_RESOURCE); + assertNotNull("the generated main class was not recorded", name); + assertEquals("com.example.BackendApplication", new String(name, "UTF-8")); + } + + @Test + public void refusesAParameterItCannotBind() throws Exception { + String source = + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Bad {\n" + + " @GetMapping(\"/x\")\n" + + " public String x(String unannotated) { return unannotated; }\n" + + "}\n"; + ProcessorContext ctx = run(compile(source)); + assertTrue("an unbindable parameter must be reported, not guessed at", + ctx.hasErrors()); + } + + @Test + public void refusesAPathVariableThatIsNotInTheRoute() throws Exception { + String source = + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Bad {\n" + + " @GetMapping(\"/x/{id}\")\n" + + " public String x(@PathVariable(\"other\") String id) { return id; }\n" + + "}\n"; + assertTrue(run(compile(source)).hasErrors()); + } + + // ---------------------------------------------------------------- + + /// The generated router, loaded and callable. + /// A second controller for the cases the first cannot express: a void route, + /// a parameter that is explicitly optional, and a required body. + private static final String OPTIONAL_SOURCE = + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import java.util.*;\n" + + "@RestController\n" + + "@RequestMapping(\"/api\")\n" + + "public class Notes {\n" + + " @DeleteMapping(\"/notes/{id}\")\n" + + " public void remove(@PathVariable(\"id\") String id) { }\n" + + " @GetMapping(\"/opt\")\n" + + " public String opt(@RequestParam(value=\"q\", required=false) String q) {\n" + + " return q == null ? \"none\" : q;\n" + + " }\n" + + " @PostMapping(\"/notes\")\n" + + " public String create(@RequestBody String body) { return body; }\n" + + "}\n"; + + @Test + public void aVoidRouteAnswersNoContent() throws Exception { + Router router = generate(OPTIONAL_SOURCE); + Object response = router.call("DELETE", "/api/notes/42", null); + assertNotNull("DELETE /api/notes/42 matched no route", response); + // ResponseStatus documents this default; the generator used to answer 200 + // for a void method, which made that javadoc wrong. + assertEquals(204, Router.statusOf(response)); + } + + @Test + public void anAbsentRequiredParamIsRefused() throws Exception { + Router router = generate(CONTROLLER_SOURCE); + // q is @RequestParam("q"), so required defaults to true. + Object missing = router.call("GET", "/api/search", null); + assertNotNull("GET /api/search matched no route", missing); + assertEquals(400, Router.statusOf(missing)); + assertTrue(Router.bodyOf(missing), Router.bodyOf(missing).indexOf("q") >= 0); + // and the route still works when it is supplied + assertEquals("{\"q\":\"hi\",\"page\":7}", router.text("GET", "/api/search?q=hi")); + } + + @Test + public void anAbsentRequiredHeaderIsRefused() throws Exception { + Router router = generate(CONTROLLER_SOURCE); + // This harness builds a Request with an empty header index, so no header + // is bindable through it -- which makes it exactly the "the client did + // not send it" case that @RequestHeader's required element describes. + Object response = router.call("GET", "/api/agent", null); + assertNotNull("GET /api/agent matched no route", response); + assertEquals(400, Router.statusOf(response)); + assertTrue(Router.bodyOf(response), + Router.bodyOf(response).indexOf("user-agent") >= 0); + } + + @Test + public void anOptionalParamIsStillOptional() throws Exception { + Router router = generate(OPTIONAL_SOURCE); + // required=false, so its absence is not an error -- the guard must not + // have been emitted for it. + assertEquals("none", router.text("GET", "/api/opt")); + assertEquals("hi", router.text("GET", "/api/opt?q=hi")); + } + + @Test + public void anAbsentRequiredBodyIsRefused() throws Exception { + Router router = generate(OPTIONAL_SOURCE); + Object response = router.call("POST", "/api/notes", null); + assertNotNull("POST /api/notes matched no route", response); + assertEquals(400, Router.statusOf(response)); + assertEquals("body", router.text2("POST", "/api/notes", "body")); + } + + @Test + public void twoRoutesOfTheSameShapeAreRefused() throws Exception { + // The variable names differ; nothing a request carries does. The second + // method could never have run. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes/{id}\")\n" + + " public String byId(@PathVariable(\"id\") String id) { return id; }\n" + + " @GetMapping(\"/notes/{name}\")\n" + + " public String byName(@PathVariable(\"name\") String name) { return name; }\n" + + "}\n")); + assertTrue("a shape that can never match should not compile", ctx.hasErrors()); + String all = ctx.getErrors().toString(); + assertTrue(all, all.indexOf("can never run") >= 0); + } + + @Test + public void aDefaultThatIsNotOfTheTypeIsRefused() throws Exception { + // to answers its fallback for anything unparseable, so this bound + // 0 -- and a non-empty default also skips the required-value guard, so an + // absent parameter reached the handler as a number nobody wrote. It is + // the author's own configuration, so it is wrong at build time or never. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all(@RequestParam(value = \"limit\", " + + "defaultValue = \"oops\") int limit) { return \"[]\"; }\n" + + "}\n")); + assertTrue("a default that is not an int should not compile", ctx.hasErrors()); + assertTrue(ctx.getErrors().toString(), + ctx.getErrors().toString().indexOf("silently replaced by zero") >= 0); + } + + @Test + public void aWellFormedDefaultStillCompiles() throws Exception { + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all(@RequestParam(value = \"limit\", " + + "defaultValue = \"20\") int limit) { return \"[]\"; }\n" + + "}\n")); + assertTrue("a valid default must still compile: " + ctx.getErrors(), + !ctx.hasErrors()); + } + + @Test + public void anEmptyNumericValueIsRejectedRatherThanZero() throws Exception { + // "?limit=" is a parameter the client SENT. Treating it as an omission + // bound the default, so the handler ran on a number nobody wrote -- the + // same defect as accepting "zz", which is already a 400. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all(@RequestParam(value = \"limit\", " + + "defaultValue = \"20\") int limit) { return String.valueOf(limit); }\n" + + "}\n"); + Object empty = router.call("GET", "/notes?limit=", null); + assertNotNull("GET /notes matched no route", empty); + assertEquals(400, Router.statusOf(empty)); + // Omitting it entirely still takes the declared default. + Object absent = router.call("GET", "/notes", null); + assertNotNull(absent); + assertEquals(200, Router.statusOf(absent)); + assertEquals("20", Router.bodyOf(absent)); + } + + @Test + public void aBodyElementOfTheWrongTypeIs400NotACrash() throws Exception { + // Declaring List does not make the elements strings. "[1]" fills + // it with a Long, and the handler's first read as a String throws -- + // answering 500 to what is really a malformed request. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import java.util.List;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @PostMapping(\"/notes\")\n" + + " public String add(@RequestBody List body) {\n" + + " return body.isEmpty() ? \"\" : body.get(0);\n" + + " }\n" + + "}\n"); + Object wrong = router.call("POST", "/notes", "[1]"); + assertNotNull("POST /notes matched no route", wrong); + assertEquals(400, Router.statusOf(wrong)); + // The declared shape still works. + Object right = router.call("POST", "/notes", "[\"hi\"]"); + assertNotNull(right); + assertEquals(200, Router.statusOf(right)); + assertEquals("hi", Router.bodyOf(right)); + } + + @Test + public void aNestedBodyElementOfTheWrongTypeIsAlso400() throws Exception { + // The one-level check stopped at the outer list, because a nested + // container is not one of the scalar types it looked for. "[[1]]" then + // reached the handler with a Long where the inner list promised a String. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import java.util.List;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @PostMapping(\"/rows\")\n" + + " public String add(@RequestBody List> rows) {\n" + + " return rows.isEmpty() || rows.get(0).isEmpty() ? \"\" " + + ": rows.get(0).get(0);\n" + + " }\n" + + "}\n"); + Object wrong = router.call("POST", "/rows", "[[1]]"); + assertNotNull("POST /rows matched no route", wrong); + assertEquals(400, Router.statusOf(wrong)); + Object right = router.call("POST", "/rows", "[[\"hi\"]]"); + assertNotNull(right); + assertEquals(200, Router.statusOf(right)); + assertEquals("hi", Router.bodyOf(right)); + } + + @Test + public void aGetRouteAnswersHeadUnlessOneIsDeclared() throws Exception { + // A HEAD asks what a GET would answer, and the server's writer already + // suppresses the body -- so a controller with only @GetMapping used to + // answer 404 to every HEAD, which breaks health checks and cache probes. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all() { return \"[]\"; }\n" + + "}\n"); + Object head = router.call("HEAD", "/notes", null); + assertNotNull("HEAD /notes matched no route", head); + assertEquals(200, Router.statusOf(head)); + assertEquals(200, Router.statusOf(router.call("GET", "/notes", null))); + } + + @Test + public void anExplicitHeadRouteWinsOverTheGetFallback() throws Exception { + // The fallback must not swallow the specific case it defers to. Sorted + // alphabetically GET comes first, so declaring both would have made the + // HEAD route unreachable the moment the fallback was added. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all() { return \"from-get\"; }\n" + + " @RequestMapping(value = \"/notes\", method = \"HEAD\")\n" + + " @ResponseStatus(204)\n" + + " public void probe() { }\n" + + "}\n"); + // JUnit 4 order: message first. + assertEquals("the declared HEAD route must win over the GET fallback", + 204, Router.statusOf(router.call("HEAD", "/notes", null))); + Object get = router.call("GET", "/notes", null); + assertEquals(200, Router.statusOf(get)); + assertEquals("from-get", Router.bodyOf(get)); + } + + @Test + public void adjacentPathVariablesAreRefused() throws Exception { + // Nothing separates them, so the matcher hands the first variable the + // whole remainder and then fails because a second is still owed: the + // route compiled and answered 404 to every request, which is the worst + // way to be wrong -- the build says fine and the endpoint does not exist. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/{left}{right}\")\n" + + " public String both(@PathVariable(\"left\") String left,\n" + + " @PathVariable(\"right\") String right) { return left; }\n" + + "}\n")); + assertTrue("adjacent variables should not compile", ctx.hasErrors()); + assertTrue(ctx.getErrors().toString(), + ctx.getErrors().toString().indexOf("adjacent") >= 0); + } + + @Test + public void variablesSeparatedByALiteralStillRoute() throws Exception { + // The separated form is the one people write, and it has to keep working. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/{left}-{right}\")\n" + + " public String both(@PathVariable(\"left\") String left,\n" + + " @PathVariable(\"right\") String right) {\n" + + " return left + \"|\" + right;\n" + + " }\n" + + "}\n"); + Object response = router.call("GET", "/a-b", null); + assertNotNull("GET /a-b matched no route", response); + assertEquals("a|b", Router.bodyOf(response)); + } + + @Test + public void anEmptyBooleanValueIsRejectedRatherThanFalse() throws Exception { + // "?enabled=" is a parameter the client SENT. Binding it to false hands + // the controller a decision nobody made -- the same defect the numeric + // bindings were fixed for, one type over. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/flag\")\n" + + " public String flag(@RequestParam(value = \"enabled\", " + + "defaultValue = \"true\") boolean enabled) { return String.valueOf(enabled); }\n" + + "}\n"); + assertEquals(400, Router.statusOf(router.call("GET", "/flag?enabled=", null))); + // Omitted entirely still takes the declared default, and a real value works. + Object absent = router.call("GET", "/flag", null); + assertEquals(200, Router.statusOf(absent)); + assertEquals("true", Router.bodyOf(absent)); + assertEquals(200, Router.statusOf(router.call("GET", "/flag?enabled=false", null))); + } + + @Test + public void aFloatThatOverflowsDoubleIsAlsoRejected() throws Exception { + // The guard tested the PARSED value for infinity, but parseDouble("1e999") + // is itself infinite -- so every double-overflowing value looked like a + // deliberate "Infinity" and was handed to the controller. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/scale\")\n" + + " public String scale(@RequestParam(\"f\") float f) { return String.valueOf(f); }\n" + + "}\n"); + assertEquals(400, Router.statusOf(router.call("GET", "/scale?f=1e999", null))); + assertEquals(400, Router.statusOf(router.call("GET", "/scale?f=1e100", null))); + assertEquals(200, Router.statusOf(router.call("GET", "/scale?f=1.5", null))); + } + + @Test + public void deeplyNestedBodyElementsAreCheckedAtEveryLevel() throws Exception { + // The emitter used to stop below the fifth level while the build-time + // rule accepted the whole shape, so a declaration nested deeper was + // checked partway and the rest reached the handler unverified. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import java.util.List;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @PostMapping(\"/deep\")\n" + + " public String add(@RequestBody " + + "List>>>>> deep) { return \"ok\"; }\n" + + "}\n"); + // A number at the innermost string position, six levels down. + Object wrong = router.call("POST", "/deep", "[[[[[[1]]]]]]"); + assertNotNull("POST /deep matched no route", wrong); + assertEquals(400, Router.statusOf(wrong)); + Object right = router.call("POST", "/deep", "[[[[[[\"hi\"]]]]]]"); + assertNotNull(right); + assertEquals(200, Router.statusOf(right)); + } + + @Test + public void aDoubleTooLargeForADoubleIsRejected() throws Exception { + // parseDouble answers INFINITY for 1e999 rather than throwing, so the + // guard approved it and the handler ran on an infinite amount -- which + // Json then writes back as null, giving the client neither its value nor + // an error. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/amount\")\n" + + " public String amount(@RequestParam(\"d\") double d) { return String.valueOf(d); }\n" + + "}\n"); + Object tooLarge = router.call("GET", "/amount?d=1e999", null); + assertNotNull("GET /amount matched no route", tooLarge); + assertEquals(400, Router.statusOf(tooLarge)); + Object ok = router.call("GET", "/amount?d=1.5", null); + assertNotNull(ok); + assertEquals(200, Router.statusOf(ok)); + } + + @Test + public void aFloatTooLargeForAFloatIsRejected() throws Exception { + // Float.parseFloat answers INFINITY for 1e100 rather than throwing, so + // the guard approved it and the handler ran on a number the client did + // not send. Every other width throws and was already refused. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/scale\")\n" + + " public String scale(@RequestParam(\"f\") float f) { return String.valueOf(f); }\n" + + "}\n"); + Object tooLarge = router.call("GET", "/scale?f=1e100", null); + assertNotNull("GET /scale matched no route", tooLarge); + assertEquals(400, Router.statusOf(tooLarge)); + // One that fits is still served. + Object ok = router.call("GET", "/scale?f=1.5", null); + assertNotNull(ok); + assertEquals(200, Router.statusOf(ok)); + } + + @Test + public void aRelativeClassPrefixStillRoutes() throws Exception { + // Written without the leading slash, which is the ordinary slip. Every + // request target has one, so the route has to as well or nothing can + // ever match it and the endpoint answers 404 while the build says fine. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "@RequestMapping(\"api\")\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all() { return \"[]\"; }\n" + + "}\n"); + Object response = router.call("GET", "/api/notes", null); + assertNotNull("GET /api/notes matched no route", response); + assertEquals("[]", Router.bodyOf(response)); + } + + @Test + public void aBodyOfDtosIsRefused() throws Exception { + // The descriptor erases this to java.util.List, which binds. What the + // parser actually supplies is a list of Map, so the first use of an + // element as a Note throws and the endpoint answers 500 -- having + // packaged perfectly. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import java.util.List;\n" + + "class Note { public String title = \"t\"; }\n" + + "@RestController\n" + + "public class Notes {\n" + + " @PostMapping(\"/notes\")\n" + + " public String add(@RequestBody List body) { return \"ok\"; }\n" + + "}\n")); + assertTrue("a body of DTOs should not compile", ctx.hasErrors()); + assertTrue(ctx.getErrors().toString(), + ctx.getErrors().toString().indexOf("Cannot bind") >= 0); + } + + @Test + public void aBodyOfMapsIsStillAllowed() throws Exception { + // What the parser really produces, so it has to keep working. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import java.util.List;\n" + + "import java.util.Map;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @PostMapping(\"/notes\")\n" + + " public String add(@RequestBody List body) { return \"ok\"; }\n" + + "}\n")); + assertTrue("List is what the parser produces: " + ctx.getErrors(), + !ctx.hasErrors()); + } + + @Test + public void aResponseInsideACollectionIsRefused() throws Exception { + // Returning a Response IS how a route answers, and emitRoute sends it. + // Inside a collection nothing does: it reaches Json's fallback and comes + // back as the quoted result of its toString(). The exemption is real at + // the top and false one level in. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import com.codename1.backend.HttpServer;\n" + + "import java.util.List;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/many\")\n" + + " public List many() { return null; }\n" + + "}\n")); + assertTrue("a collection of Response should not compile", ctx.hasErrors()); + assertTrue(ctx.getErrors().toString(), + ctx.getErrors().toString().indexOf("cannot encode") >= 0); + } + + @Test + public void aDirectResponseReturnIsSentAsItStands() throws Exception { + // Not just that it compiles: that the router SENDS it. The comparison + // this branch turns on used the dotted source spelling of a nested class + // while the type comes from the descriptor as HttpServer$Response, so it + // never matched -- the branch that sends a Response was dead, and a + // controller taking control of its own reply had that reply JSON-encoded + // instead. Status 418 is the reply here; a JSON-encoded one would be 200. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import com.codename1.backend.HttpServer;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/one\")\n" + + " public HttpServer.Response one() {\n" + + " return HttpServer.Response.text(418, \"teapot\");\n" + + " }\n" + + "}\n"); + Object response = router.call("GET", "/one", null); + assertNotNull("GET /one matched no route", response); + assertEquals(418, Router.statusOf(response)); + assertEquals("teapot", Router.bodyOf(response)); + } + + @Test + public void aJdkReturnJsonCannotWriteIsRefused() throws Exception { + // java.util.Date has no branch in Json.writeValue, so it reaches the + // final one and is answered as a quoted, implementation-formatted + // toString() -- a date the client cannot parse back, from a build and a + // request that both reported success. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/when\")\n" + + " public java.util.Date when() { return null; }\n" + + "}\n")); + assertTrue("a JDK type Json cannot write should not compile", ctx.hasErrors()); + assertTrue(ctx.getErrors().toString(), + ctx.getErrors().toString().indexOf("cannot encode") >= 0); + } + + @Test + public void aStatusOutsideTheHttpRangeIsRefused() throws Exception { + // Copied verbatim into the router, emitted verbatim as the status line + // and as :status -- so a typo turns a working handler into a reply the + // client rejects or cannot frame. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/oops\")\n" + + " @ResponseStatus(700)\n" + + " public String oops() { return \"x\"; }\n" + + "}\n")); + assertTrue("a status outside 200..599 should not compile", ctx.hasErrors()); + assertTrue(ctx.getErrors().toString(), + ctx.getErrors().toString().indexOf("between 200 and 599") >= 0); + } + + @Test + public void anInformationalStatusIsRefused() throws Exception { + // In range for HTTP, but not an ANSWER: a generated route sends one + // response, and a 1xx is interim -- the client waits for a final one + // that never comes, and the writer ends the response at the headers. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/interim\")\n" + + " @ResponseStatus(102)\n" + + " public String interim() { return \"x\"; }\n" + + "}\n")); + assertTrue("an interim status should not compile", ctx.hasErrors()); + assertTrue(ctx.getErrors().toString(), + ctx.getErrors().toString().indexOf("between 200 and 599") >= 0); + } + + @Test + public void anArrayReturnOtherThanBytesIsRefused() throws Exception { + // Json writes byte[] as base64 and has no handling for any other array, + // so this would be answered as the JSON string "[I@1a2b3c" while the + // build and the request both reported success. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/ids\")\n" + + " public int[] ids() { return new int[0]; }\n" + + "}\n")); + assertTrue("an array the router cannot encode should not compile", ctx.hasErrors()); + String all = ctx.getErrors().toString(); + assertTrue(all, all.indexOf("cannot encode") >= 0); + } + + @Test + public void aByteArrayReturnIsStillAllowed() throws Exception { + // The one array shape Json does handle: base64, deliberately. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/blob\")\n" + + " public byte[] blob() { return new byte[0]; }\n" + + "}\n")); + assertTrue("byte[] is encodable and must still compile: " + ctx.getErrors(), + !ctx.hasErrors()); + } + + @Test + public void aLiteralAndAVariableInOneControllerBothWork() throws Exception { + // The single most ordinary pair there is. They DO overlap -- /users/me is + // a path /users/{id} would answer -- but the router emits every route + // with no variables before any route with one, so the literal wins its + // own path and everything else falls through. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/users/me\")\n" + + " public String me() { return \"me\"; }\n" + + " @GetMapping(\"/users/{id}\")\n" + + " public String byId(@PathVariable(\"id\") String id) { return id; }\n" + + "}\n"); + Object mine = router.call("GET", "/users/me", null); + assertNotNull("GET /users/me matched no route", mine); + assertEquals("me", Router.bodyOf(mine)); + Object other = router.call("GET", "/users/42", null); + assertNotNull("GET /users/42 matched no route", other); + assertEquals("42", Router.bodyOf(other)); + } + + @Test + public void aListOfDtosIsRefused() throws Exception { + // The DESCRIPTOR erases this to java.util.List, which the encodable + // check waves through on its own name. Every Note in the list would + // then be written as the quoted result of its toString(), while the + // build and the request both reported success. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "import java.util.List;\n" + + "class Note { public String title = \"t\"; }\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public List all() { return null; }\n" + + "}\n")); + assertTrue("a list of types the router cannot encode should not compile", + ctx.hasErrors()); + String all = ctx.getErrors().toString(); + assertTrue(all, all.indexOf("cannot encode") >= 0); + } + + @Test + public void aVerbTheServerDoesNotRouteIsRefused() throws Exception { + // HttpServer compares the verb with equals and answers 501 before + // dispatch, so this route could never be reached -- and nothing said so: + // the build passed and the endpoint simply did not exist. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @RequestMapping(value = \"/notes\", method = \"get\")\n" + + " public String all() { return \"[]\"; }\n" + + "}\n")); + assertTrue("a verb the server cannot route should not compile", ctx.hasErrors()); + String all = ctx.getErrors().toString(); + assertTrue(all, all.indexOf("does not route") >= 0); + } + + @Test + public void aBodyThatIsNotJsonIsRefused() throws Exception { + Router router = generate(CONTROLLER_SOURCE); + // bodyAsMap answers null both for "no body" and for "not JSON", so the + // controller used to be called with null and the client saw a 404, a 500, + // or a side effect performed on an argument it never sent. + Object bad = router.call("POST", "/api/notes", "{not json"); + assertNotNull("POST /api/notes matched no route", bad); + assertEquals(400, Router.statusOf(bad)); + // A body that is valid JSON still reaches the handler with its status. + Object good = router.call("POST", "/api/notes", "{\"a\":1}"); + assertNotNull(good); + assertEquals(201, Router.statusOf(good)); + } + + @Test + public void aVariableMayContainTheLiteralThatFollowsIt() throws Exception { + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/download/{name}.json\")\n" + + " public String get(@PathVariable(\"name\") String name) { return name; }\n" + + "}\n"); + assertEquals("foo", router.text("GET", "/download/foo.json")); + // The value itself ends in the literal. Taking the first occurrence left + // ".json" unconsumed and rejected a request this route does match. + assertEquals("foo.json", router.text("GET", "/download/foo.json.json")); + // Still one segment, and still anchored at the end. + assertNull(router.call("GET", "/download/a/b.json", null)); + assertNull(router.call("GET", "/download/foo.jsonx", null)); + } + + @Test + public void processingTwiceWithoutCleaningStillWorks() throws Exception { + // The second pass of an incremental build scans target/classes, which by + // then contains the FIRST pass's NotesRouter. The collision check looked it + // up unconditionally and reported the processor's own output as a + // user-defined class it would overwrite -- so every project using + // @RestController failed its second `mvn process-classes` and only a clean + // could get it building again. + File classes = compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all() { return \"[]\"; }\n" + + "}\n"); + ProcessorContext first = run(classes); + assertFalse("the first pass should be clean: " + first.getErrors(), + first.hasErrors()); + assertTrue("the first pass must have written the router it then trips over", + new File(classes, "com/example/NotesRouter.class").isFile()); + + ProcessorContext second = run(classes); + assertFalse("processing twice without a clean must work: " + second.getErrors(), + second.hasErrors()); + } + + @Test + public void aMalformedEscapeIsA400AndNotALiteral() throws Exception { + // %ZZ is not a character, and decoding it as the three literal characters + // handed the controller a value no client can have meant. Worse, it + // ALIASES: %252F is a correctly escaped %2F, and if a bad escape passes + // through as text then a check written against one spelling is defeated + // by the other. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes/{id}\")\n" + + " public String byId(@PathVariable(\"id\") String id) { return \"id=\" + id; }\n" + + "}\n"); + // A GOOD escape still decodes: %41 is 'A'. + assertEquals("id=A", router.text("GET", "/notes/%41")); + + // JUnit 4: message first. + assertEquals("a non-hex escape is a syntax error the client can fix, so 400", + 400, Router.statusOf(router.call("GET", "/notes/%ZZ", null))); + assertEquals("and so is a truncated one", + 400, Router.statusOf(router.call("GET", "/notes/%2", null))); + } + + @Test + public void twoControllersOfTheSameShapeAreRefused() throws Exception { + // The bootstrap chains the routers and returns the first non-null answer, + // so a collision ACROSS controllers hides the later one exactly as a + // collision inside one does. Checking each controller alone missed it. + ProcessorContext ctx = run(compileBoth( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes/{id}\")\n" + + " public String byId(@PathVariable(\"id\") String id) { return id; }\n" + + "}\n", + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Other {\n" + + " @GetMapping(\"/notes/{name}\")\n" + + " public String byName(@PathVariable(\"name\") String name) { return name; }\n" + + "}\n")); + assertTrue("a shape claimed by two controllers should not compile", ctx.hasErrors()); + String all = ctx.getErrors().toString(); + assertTrue(all, all.indexOf("can never run") >= 0); + } + + @Test + public void aGetInOneControllerAndAHeadInAnotherAreRefused() throws Exception { + // A generated GET block also answers HEAD, so these two DO compete even + // though the verbs differ -- and across controllers nothing orders them: + // the bootstrap tries the routers in turn, so whichever it lists first + // takes the HEAD and the declared handler never runs. Inside ONE + // controller the same pair is fine, because the comparator emits HEAD's + // block ahead of GET's fallback. + ProcessorContext ctx = run(compileBoth( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all() { return \"[]\"; }\n" + + "}\n", + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Other {\n" + + " @RequestMapping(value = \"/notes\", method = \"HEAD\")\n" + + " public void probe() { }\n" + + "}\n")); + assertTrue("a HEAD hidden by another controller's GET should not compile", + ctx.hasErrors()); + } + + @Test + public void aGetAndAHeadInTheSameControllerStillCompile() throws Exception { + // The other side of that rule. Making the pair collide across controllers + // must not make the ordinary declaration -- both in one class, which is + // what the cross-controller message tells people to do -- unwritable. + ProcessorContext ctx = run(compileBoth( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public String all() { return \"[]\"; }\n" + + " @RequestMapping(value = \"/notes\", method = \"HEAD\")\n" + + " public void probe() { }\n" + + "}\n", + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Other {\n" + + " @GetMapping(\"/other\")\n" + + " public String other() { return \"x\"; }\n" + + "}\n")); + assertFalse("GET and HEAD in one controller are ordered, not ambiguous: " + + ctx.getErrors(), ctx.hasErrors()); + } + + @Test + public void aMapBodyKeyedByANonStringIsRefused() throws Exception { + // A JSON object's names are always strings. Long is a fine body VALUE -- + // every JSON integer arrives as one -- so the element rule approved + // Map, and the emitted shape check walks values() only. The + // handler then got a map whose keys violate its own declaration: typed + // iteration throws, and get(1L) misses the value the client sent. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Counts {\n" + + " @PostMapping(\"/counts\")\n" + + " public String put(@RequestBody java.util.Map counts) {\n" + + " return \"ok\";\n" + + " }\n" + + "}\n")); + assertTrue("a map keyed by Long cannot be decoded and should not compile", + ctx.hasErrors()); + String all = ctx.getErrors().toString(); + assertTrue(all, all.indexOf("names are strings") >= 0); + } + + @Test + public void routesWithDisjointSuffixesAreNotAmbiguous() throws Exception { + // No request satisfies both: one ends .json, the other .xml. The overlap + // check treated every pair of variable-carrying segments as colliding, so + // a controller that cannot be ambiguous failed to compile -- and the + // matcher it would have generated handles literals around a variable + // perfectly well. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/{name}.json\")\n" + + " public String json(@PathVariable(\"name\") String name) {\n" + + " return \"json:\" + name;\n" + + " }\n" + + " @GetMapping(\"/{name}.xml\")\n" + + " public String xml(@PathVariable(\"name\") String name) {\n" + + " return \"xml:\" + name;\n" + + " }\n" + + "}\n"); + assertEquals("json:a", router.text("GET", "/a.json")); + assertEquals("xml:a", router.text("GET", "/a.xml")); + } + + @Test + public void routesWithOverlappingSuffixesAreStillRefused() throws Exception { + // And the check must still bite where the two CAN collide: a bare + // variable matches "a.json" as readily as {name}.json does. + ProcessorContext ctx = run(compileBoth( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/{name}.json\")\n" + + " public String json(@PathVariable(\"name\") String name) { return name; }\n" + + "}\n", + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Other {\n" + + " @GetMapping(\"/{anything}\")\n" + + " public String any(@PathVariable(\"anything\") String a) { return a; }\n" + + "}\n")); + assertTrue("a bare variable answers /a.json too, so these do collide", + ctx.hasErrors()); + } + + @Test + public void aMapReturnKeyedByANonStringIsRefused() throws Exception { + // Json.writeValue calls String.valueOf on every map key whatever it is, + // so the keys come back as object identity -- "[B@1a2b3c" -- which is + // different on every run and describes nothing. The key was being checked + // as though it were a value, and byte[] is a perfectly good value. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Blobs {\n" + + " @GetMapping(\"/blobs\")\n" + + " public java.util.Map all() { return null; }\n" + + "}\n")); + assertTrue("a map keyed by byte[] cannot be written as JSON", ctx.hasErrors()); + } + + @Test + public void aMapReturnKeyedByStringIsAccepted() throws Exception { + // The shape the rule is protecting has to keep working. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Counts {\n" + + " @GetMapping(\"/counts\")\n" + + " public java.util.Map all() { return null; }\n" + + "}\n")); + assertFalse("Map is exactly what Json writes: " + ctx.getErrors(), + ctx.hasErrors()); + } + + @Test + public void anUnboundedWildcardReturnElementIsRefused() throws Exception { + // "?" is not a primitive, it is UNKNOWN. Reaching the no-dot branch it was + // read as one, so List was approved and a handler returning a DTO or a + // Date inside it got Json's quoted toString() fallback -- the malformed + // contract this validation exists to refuse for List. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @GetMapping(\"/notes\")\n" + + " public java.util.List all() { return null; }\n" + + "}\n")); + assertTrue("a List return says nothing about what Json must write", + ctx.hasErrors()); + } + + @Test + public void aBoundedWildcardElementIsCheckedLikeItsBound() throws Exception { + // List was accepted with NO runtime element check at + // all, because every consumer read a bounded wildcard as "claims + // nothing". The bound is a claim: a body of [1] reached the handler as a + // list holding a Long, and the first typed read answered 500 where a 400 + // was owed. Normalising the wildcard where type arguments are produced + // fixes the validation and the emitted check together. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @PostMapping(\"/notes\")\n" + + " public String add(@RequestBody" + + " java.util.List notes) {\n" + + " return \"ok\";\n" + + " }\n" + + "}\n"); + assertEquals(200, Router.statusOf(router.call("POST", "/notes", "[\"a\"]"))); + // JUnit 4 order: message first. + assertEquals("a Long where the bound promised String is the client's mistake, " + + "so it is a 400 and not a 500", + 400, Router.statusOf(router.call("POST", "/notes", "[1]"))); + } + + @Test + public void aSuperBoundedWildcardElementIsNotChecked() throws Exception { + // The other direction, and it must NOT be normalised the same way: + // List allows a String or any supertype, so an element + // check against String would reject values the declaration permits. + Router router = generate( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Notes {\n" + + " @PostMapping(\"/notes\")\n" + + " public String add(@RequestBody" + + " java.util.List notes) {\n" + + " return \"ok\";\n" + + " }\n" + + "}\n"); + assertEquals("? super String permits a Long element, so nothing may reject it", + 200, Router.statusOf(router.call("POST", "/notes", "[1]"))); + } + + @Test + public void aMapBodyKeyedByABoundedWildcardIsRefused() throws Exception { + // "? extends Long" is not the same claim as "?". The bound still promises + // every key is a Long, so `for (Long key : body.keySet())` compiles and + // then meets the Strings a JSON object really produces -- a 500 for what + // is a 400. Exempting everything starting with '?' let it through. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Bounded {\n" + + " @PostMapping(\"/counts\")\n" + + " public String put(@RequestBody" + + " java.util.Map counts) {\n" + + " return \"ok\";\n" + + " }\n" + + "}\n")); + assertTrue("a bounded wildcard key is still a promise about the key type", + ctx.hasErrors()); + } + + @Test + public void aMapBodyKeyedByAnUnboundedWildcardIsAccepted() throws Exception { + // The unbounded one claims nothing, so it stays legal -- the rule must + // separate "any key" from "a Long key spelled as a wildcard". + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Unbounded {\n" + + " @PostMapping(\"/counts\")\n" + + " public String put(@RequestBody java.util.Map counts) {\n" + + " return \"ok\";\n" + + " }\n" + + "}\n")); + assertFalse("Map claims nothing about its keys: " + ctx.getErrors(), + ctx.hasErrors()); + } + + @Test + public void aMapBodyKeyedByStringIsAccepted() throws Exception { + // The rule must not swallow the shape it is protecting. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Counts {\n" + + " @PostMapping(\"/counts\")\n" + + " public String put(@RequestBody java.util.Map counts) {\n" + + " return \"ok\";\n" + + " }\n" + + "}\n")); + assertFalse("Map is exactly what a JSON object decodes to: " + + ctx.getErrors(), ctx.hasErrors()); + } + + @Test + public void anOverflowingDoubleDefaultIsRefused() throws Exception { + // Double.parseDouble("1e999") answers infinity instead of throwing, so + // this declaration was approved while the RUNTIME guard rejects the same + // spelling arriving in a request: omit the parameter and the controller + // runs on an infinity, send it and the client gets a 400. Two answers for + // one value. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Rates {\n" + + " @GetMapping(\"/rate\")\n" + + " public String rate(@RequestParam(value = \"r\", defaultValue = \"1e999\")\n" + + " double r) { return String.valueOf(r); }\n" + + "}\n")); + assertTrue("a default that parses to infinity should not compile", ctx.hasErrors()); + } + + @Test + public void anExplicitInfinityDefaultIsStillAllowed() throws Exception { + // The spelling is what says an infinity was meant, which is the same test + // the generated guard uses -- so the two cannot disagree about which + // values are infinities. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Rates {\n" + + " @GetMapping(\"/rate\")\n" + + " public String rate(@RequestParam(value = \"r\", defaultValue = \"Infinity\")\n" + + " double r) { return String.valueOf(r); }\n" + + "}\n")); + assertFalse("a deliberate Infinity is not an overflow: " + ctx.getErrors(), + ctx.hasErrors()); + } + + @Test + public void anOverflowingFloatDefaultIsRefused() throws Exception { + // The float branch had the bug in the other direction: Double.isInfinite + // was its "did they mean it" test, and Double.parseDouble("1e999") is + // itself infinite, so every double-overflowing default read as deliberate. + ProcessorContext ctx = run(compile( + "package com.example;\n" + + "import com.codename1.backend.annotations.*;\n" + + "@RestController\n" + + "public class Rates {\n" + + " @GetMapping(\"/rate\")\n" + + " public String rate(@RequestParam(value = \"r\", defaultValue = \"1e999\")\n" + + " float r) { return String.valueOf(r); }\n" + + "}\n")); + assertTrue("a float default that parses to infinity should not compile", + ctx.hasErrors()); + } + + private static final class Router { + private final Object instance; + private final Method handle; + private static Constructor requestCtor; + private static Field bodyField; + private static Field deferredField; + private static Field hasDeferredField; + private static Field statusField; + + Router(Object instance, Method handle) { + this.instance = instance; + this.handle = handle; + } + + Object call(String method, String target, String body) throws Exception { + return handle.invoke(instance, request(method, target, body)); + } + + String text2(String method, String target, String body) throws Exception { + Object response = call(method, target, body); + assertNotNull(method + " " + target + " matched no route", response); + return bodyOf(response); + } + + String text(String method, String target) throws Exception { + Object response = call(method, target, null); + assertNotNull(method + " " + target + " matched no route", response); + return bodyOf(response); + } + + static int statusOf(Object response) throws Exception { + reflect(); + return statusField.getInt(response); + } + + static String bodyOf(Object response) throws Exception { + reflect(); + if (hasDeferredField.getBoolean(response)) { + // respondJson leaves the value unserialised for the writer; rendering + // it here is what the server does at write time. + return Json.write(deferredField.get(response)); + } + return new String((byte[]) bodyField.get(response), "UTF-8"); + } + + private static HttpServer.Request request(String method, String target, String body) + throws Exception { + reflect(); + byte[] raw = (method + " " + target + " HTTP/1.1\r\nUser-Agent: probe\r\n\r\n") + .getBytes("UTF-8"); + return (HttpServer.Request) requestCtor.newInstance(method, target, "HTTP/1.1", raw, + new int[0], 0, body, Integer.valueOf(method.length() + 1), + Integer.valueOf(target.getBytes("UTF-8").length)); + } + + private static synchronized void reflect() throws Exception { + if (requestCtor != null) { + return; + } + Class req = HttpServer.Request.class; + requestCtor = req.getDeclaredConstructor(String.class, String.class, String.class, + byte[].class, int[].class, int.class, String.class, int.class, int.class); + requestCtor.setAccessible(true); + Class res = HttpServer.Response.class; + bodyField = res.getDeclaredField("body"); + bodyField.setAccessible(true); + deferredField = res.getDeclaredField("deferredJson"); + deferredField.setAccessible(true); + hasDeferredField = res.getDeclaredField("hasDeferredJson"); + hasDeferredField.setAccessible(true); + statusField = res.getDeclaredField("status"); + statusField.setAccessible(true); + } + } + + private Router generate(String controllerSource) throws Exception { + File classes = compile(controllerSource); + ProcessorContext ctx = run(classes); + if (ctx.hasErrors()) { + StringBuilder sb = new StringBuilder("processor reported errors:\n"); + for (ProcessorContext.ProcessingError e : ctx.getErrors()) { + sb.append(' ').append(e).append('\n'); + } + fail(sb.toString()); + } + URLClassLoader loader = new URLClassLoader(new URL[]{ classes.toURI().toURL() }, + getClass().getClassLoader()); + Class controller = loader.loadClass("com.example.Notes"); + Class router = loader.loadClass("com.example.NotesRouter"); + Object instance = router.getConstructor(controller).newInstance(controller.newInstance()); + return new Router(instance, router.getMethod("handle", HttpServer.Request.class)); + } + + private File compile(String controllerSource) throws Exception { + File classes = tmp.newFolder(); + Map sources = new LinkedHashMap(); + // Read out of the source rather than guessed from a pair of known names: + // javac wants the file to match the class, so a test that declared a + // third name failed to COMPILE and reported that as its result. + int at = controllerSource.indexOf("public class "); + String name = controllerSource.substring(at + "public class ".length(), + controllerSource.indexOf(' ', at + "public class ".length() + 1)); + sources.put("com.example." + name.trim(), controllerSource); + JavaSourceCompiler.compile(sources, classes, backendClasspath()); + return classes; + } + + /** Compiles two controllers into one output, the way a real project has them. */ + private File compileBoth(String first, String second) throws Exception { + File classes = tmp.newFolder(); + Map sources = new LinkedHashMap(); + sources.put("com.example.Notes", first); + sources.put("com.example.Other", second); + JavaSourceCompiler.compile(sources, classes, backendClasspath()); + return classes; + } + + private ProcessorContext run(File classes) throws Exception { + Map index = ClassScanner.scan(classes); + RestControllerAnnotationProcessor proc = new RestControllerAnnotationProcessor(); + List cp = new java.util.ArrayList(); + for (File f : backendClasspath()) { + cp.add(f.getAbsolutePath()); + } + ProcessorContext ctx = new ProcessorContext(classes, tmp.newFolder(), index, + new SystemStreamLog(), tmp.newFolder(), new Properties(), null, + Collections.emptyList(), "UTF-8", cp); + proc.start(ctx); + for (AnnotatedClass cls : index.values()) { + if (!cls.getClassAnnotations().isEmpty()) { + proc.processClass(cls, ctx); + } + } + proc.finish(ctx); + return ctx; + } + + /// Where the backend runtime the generated code names actually sits. Taken from + /// the loaded class rather than from a path, so it follows the test classpath. + private static List backendClasspath() throws Exception { + URL url = HttpServer.class.getProtectionDomain().getCodeSource().getLocation(); + return Arrays.asList(new File(url.toURI())); + } +} diff --git a/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestServerAnnotationProcessorTest.java b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestServerAnnotationProcessorTest.java new file mode 100644 index 00000000000..b73b5b67ba8 --- /dev/null +++ b/maven/codenameone-maven-plugin/src/test/java/com/codename1/maven/processors/RestServerAnnotationProcessorTest.java @@ -0,0 +1,1418 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven.processors; + +import com.codename1.maven.annotations.AnnotatedClass; +import com.codename1.maven.annotations.ClassScanner; +import com.codename1.maven.annotations.JavaSourceCompiler; +import com.codename1.maven.annotations.ProcessorContext; +import org.apache.maven.plugin.logging.SystemStreamLog; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/// Proves the server half of the shared `@RestClient` contract: one annotated +/// interface produces a synchronous server interface and a dispatcher that +/// actually routes, binds and invokes. The dispatcher is loaded and CALLED here +/// rather than merely inspected -- a generated router that compiles but routes +/// nowhere is the failure this test exists to catch. +public class RestServerAnnotationProcessorTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + @Before + public void enableServerHalf() { + System.setProperty("cn1.restServer", "true"); + } + + @After + public void disableServerHalf() { + System.clearProperty("cn1.restServer"); + } + + private static final String DTO_SOURCE = + "package com.example;\n" + + "public class Pet {\n" + + " public long id;\n" + + " public String name;\n" + + " public boolean good;\n" + + " public double weight;\n" + + " public java.util.List tags;\n" + + " public java.util.List weights;\n" + + " public Pet() {}\n" + + "}\n"; + + private static final String TAG_SOURCE = + "package com.example;\n" + + "public class Tag {\n" + + " public String label;\n" + + " public int weight;\n" + + " public Tag() {}\n" + + "}\n"; + + private static final String API_SOURCE = + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface GreeterApi {\n" + + " @GET(\"/greet/{name}\")\n" + + " void greet(@Path(\"name\") String name,\n" + + " @Query(\"loud\") String loud,\n" + + " OnComplete> callback);\n" + + " @POST(\"/echo\")\n" + + " void echo(@Body String body, OnComplete> callback);\n" + + " @GET(\"/whoami\")\n" + + " void whoami(@Header(\"X-User\") String user,\n" + + " @Cookie(\"session\") String session,\n" + + " OnComplete> callback);\n" + + " @POST(\"/pet\")\n" + + " void addPet(@Body Pet pet, OnComplete> callback);\n" + + " @GET(\"/pets\")\n" + + " void listPets(OnComplete>> callback);\n" + + " @POST(\"/weights\")\n" + + " void weights(@Body java.util.List weights,\n" + + " OnComplete> callback);\n" + + " @POST(\"/labels\")\n" + + " void labels(@Body java.util.Set labels,\n" + + " OnComplete> callback);\n" + + "}\n"; + + @Test + public void aStringBodyMustHaveArrivedAsAString() throws Exception { + // Declaring @Body String does not make the body a string. A client sending + // the number 1 or an object used to be coerced with String.valueOf, so the + // handler saw "1" or "{a=1}" as though those had been sent as JSON strings, + // instead of the IllegalArgumentException the transport turns into a 400. + File classes = compileApi(); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.GreeterApiServer"); + final Object[] received = new Object[1]; + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + if ("echo".equals(m.getName())) { + received[0] = args[0]; + return args[0]; + } + return null; + } + }); + Class dispatcherClass = loader.loadClass("com.example.GreeterApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + + dispatch.invoke(dispatcher, "POST", "/echo", null, "hello"); + assertEquals("a genuine string body must still arrive", "hello", received[0]); + + received[0] = null; + try { + dispatch.invoke(dispatcher, "POST", "/echo", null, Long.valueOf(1)); + fail("a JSON number reaching a String body should be refused, not stringified"); + } catch (java.lang.reflect.InvocationTargetException expected) { + assertTrue(String.valueOf(expected.getCause()), + expected.getCause() instanceof IllegalArgumentException); + } + assertNull("the handler must not have been called at all", received[0]); + loader.close(); + } + + @Test + public void aScalarBodyStillArrivesThroughItsTextForm() throws Exception { + // The other half of the rule above: a declared `int` body IS fed by + // rendering whatever arrived and parsing it, so that a JSON number reaching + // an int body behaves like one reaching an int query parameter. Making the + // string helper strict without splitting it broke exactly this. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.TallyApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface TallyApi {\n" + + " @POST(\"/tally\")\n" + + " void tally(@Body int count, OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.TallyApiServer"); + final Object[] received = new Object[1]; + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + received[0] = args[0]; + return "ok"; + } + }); + Class dispatcherClass = loader.loadClass("com.example.TallyApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + // What the JSON reader really produces for the body `7`. + dispatch.invoke(dispatcher, "POST", "/tally", null, Long.valueOf(7)); + assertEquals(Integer.valueOf(7), received[0]); + loader.close(); + } + + @Test + public void anOverflowingFloatingPointQueryIsRefused() throws Exception { + // parseDouble and valueOf do not FAIL on a value too large: 1e999 comes + // back as infinity, so the handler ran on a number the client never sent + // and Json wrote it back as null -- neither the value nor an error. The + // boxed helpers had no check at all, and the float one asked + // !Double.isInfinite(d), which parseDouble("1e999") already satisfies. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.RateApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface RateApi {\n" + + " @GET(\"/rate\")\n" + + " void rate(@Query(\"d\") double d, @Query(\"bd\") Double bd,\n" + + " @Query(\"f\") float f, @Query(\"bf\") Float bf,\n" + + " OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class dispatcherClass = loader.loadClass("com.example.RateApiDispatcher"); + + // Every one of the four bindings, because each reached the value by its + // own helper and only one of them was guarded at all. + String[][] cases = { + {"d", "1e999"}, {"bd", "1e999"}, {"f", "1e50"}, {"bf", "1e50"}, + }; + for (int i = 0; i < cases.length; i++) { + assertRefused(loader, dispatcherClass, cases[i][0], cases[i][1]); + } + + // And a value that really spells an infinity is still accepted, which is + // what the parse means by it -- the guard is about overflow, not about + // infinities the client asked for. + Object answered = dispatch(loader, dispatcherClass, "d", "Infinity"); + assertNotNull("a deliberate Infinity must still bind", answered); + loader.close(); + } + + private void assertRefused(URLClassLoader loader, Class dispatcherClass, + String param, String value) throws Exception { + try { + dispatch(loader, dispatcherClass, param, value); + fail(param + "=" + value + " overflows and should not reach the handler"); + } catch (java.lang.reflect.InvocationTargetException expected) { + Throwable cause = expected.getCause(); + assertTrue(param + "=" + value + " failed with " + cause, + cause instanceof NumberFormatException + || cause instanceof IllegalArgumentException); + } + } + + /** Calls the generated dispatcher with one query parameter set. */ + private Object dispatch(URLClassLoader loader, Class dispatcherClass, + String param, String value) throws Exception { + Class serverItf = loader.loadClass("com.example.RateApiServer"); + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + return "ok"; + } + }); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method d = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + return d.invoke(dispatcher, "GET", "/rate?" + param + "=" + value, null, null); + } + + @Test + public void anEmbeddedPlaceholderIsBoundLikeTheClientBindsIt() throws Exception { + // The CLIENT generator substitutes {name} anywhere in the template, so + // /files/{name}.json has always produced a working client. The server + // generator only recognised a placeholder that owned a whole segment, so + // turning server generation on reported that @Path("name") was absent and + // refused a contract that already worked -- two halves of one annotation + // disagreeing about what it means. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.FileApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface FileApi {\n" + + " @GET(\"/files/{name}.json\")\n" + + " void get(@Path(\"name\") String name,\n" + + " OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.FileApiServer"); + final Object[] seen = new Object[1]; + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + seen[0] = args[0]; + return "ok"; + } + }); + Class dispatcherClass = loader.loadClass("com.example.FileApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + + assertNotNull("the route did not match at all", + dispatch.invoke(dispatcher, "GET", "/files/report.json", null, null)); + assertEquals("the value between the literals is what binds", + "report", seen[0]); + + // The literals are part of the match, not decoration. + seen[0] = null; + assertNull("a different extension must not match", + dispatch.invoke(dispatcher, "GET", "/files/report.xml", null, null)); + assertNull("and an empty value is not a segment", + dispatch.invoke(dispatcher, "GET", "/files/.json", null, null)); + } + + @Test + public void twoPlaceholdersInOneSegmentAreRefusedWithAReason() throws Exception { + // Supporting one embedded placeholder does not mean guessing at two. + // "{a}-{b}" gives no way to decide where the first value ends, and a + // server that picks one binds something the client never meant -- so the + // developer is told, rather than left with a route that never matches. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.PairApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface PairApi {\n" + + " @GET(\"/pair/{a}-{b}\")\n" + + " void pair(@Path(\"a\") String a, @Path(\"b\") String b,\n" + + " OnComplete> callback);\n" + + "}\n"); + ProcessorContext ctx = runProcessor(compileSources(sources)); + assertTrue("two placeholders in one segment cannot be split", ctx.hasErrors()); + String all = ctx.getErrors().toString(); + assertTrue(all, all.indexOf("more than one placeholder") >= 0); + } + + @Test + public void aJsonStringWhereANumberIsDeclaredIsRefused() throws Exception { + // The value has already been TYPED by the parser here, so "7" against an + // int field is the client disagreeing with the contract -- and the + // generated client could never have produced it. Parsing it anyway left + // the handler unable to tell a number from a string that looks like one. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.Counter", + "package com.example;\n" + + "public class Counter {\n" + + " public int count;\n" + + " public Counter() {}\n" + + "}\n"); + sources.put("com.example.CountApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface CountApi {\n" + + " @POST(\"/count\")\n" + + " void put(@Body Counter c, OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Method fromMap = loader.loadClass("com.example.CounterJson").getMethod("fromMap", Map.class); + + Map asNumber = new java.util.LinkedHashMap(); + asNumber.put("count", Long.valueOf(7)); + assertEquals(7, loader.loadClass("com.example.Counter").getField("count") + .get(fromMap.invoke(null, asNumber))); + + Map asText = new java.util.LinkedHashMap(); + asText.put("count", "7"); + try { + fromMap.invoke(null, asText); + fail("a JSON string where an int is declared should be refused"); + } catch (java.lang.reflect.InvocationTargetException expected) { + assertTrue(String.valueOf(expected.getCause()), + expected.getCause() instanceof IllegalArgumentException); + } + + // An ABSENT field is still zero -- the rule is about a wrong type, not a + // missing one. + Map absent = new java.util.LinkedHashMap(); + assertEquals(0, loader.loadClass("com.example.Counter").getField("count") + .get(fromMap.invoke(null, absent))); + loader.close(); + } + + @Test + public void aTextBindingStillParsesItsText() throws Exception { + // The other half of the rule, and the reason the two paths are separate: a + // QUERY parameter really does arrive as text, so parsing it is not + // leniency, it is the only thing that could work. Tightening the JSON + // decoders must not reach this. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.QueryApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface QueryApi {\n" + + " @GET(\"/count\")\n" + + " void get(@Query(\"n\") int n, OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.QueryApiServer"); + final Object[] seen = new Object[1]; + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + seen[0] = args[0]; + return "ok"; + } + }); + Class dispatcherClass = loader.loadClass("com.example.QueryApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + dispatch.invoke(dispatcher, "GET", "/count?n=7", null, null); + assertEquals(Integer.valueOf(7), seen[0]); + loader.close(); + } + + @Test + public void processingAContractTwiceWithoutCleaningStillWorks() throws Exception { + // Same rule as the controller half: the second pass of an incremental + // build scans target/classes and finds the ApiServer, ApiDispatcher and + // DTO codecs the first pass wrote. Reported as existing application + // classes, an unchanged contract could be processed exactly once per + // clean output directory. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.NoteApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface NoteApi {\n" + + " @GET(\"/notes\")\n" + + " void all(OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext first = runProcessor(classes); + assertNoErrors(first); + assertTrue("the first pass must have written the dispatcher it then trips over", + new File(classes, "com/example/NoteApiDispatcher.class").isFile()); + + ProcessorContext second = runProcessor(classes); + assertFalse("processing twice without a clean must work: " + second.getErrors(), + second.hasErrors()); + } + + @Test + public void aRealCollisionIsStillRefused() throws Exception { + // The marker must not turn the guard off. A class the DEVELOPER wrote with + // the generated name carries no marker, and generating over it would + // silently replace their code in the output directory. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.NoteApiDispatcher", + "package com.example;\n" + + "public class NoteApiDispatcher {\n" + + " public String mine() { return \"handwritten\"; }\n" + + "}\n"); + sources.put("com.example.NoteApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface NoteApi {\n" + + " @GET(\"/notes\")\n" + + " void all(OnComplete> callback);\n" + + "}\n"); + ProcessorContext ctx = runProcessor(compileSources(sources)); + assertTrue("a hand-written class of that name must still be protected", + ctx.hasErrors()); + String all = ctx.getErrors().toString(); + assertTrue(all, all.indexOf("already exists") >= 0); + } + + @Test + public void disjointSuffixesAreNotTheSameShape() throws Exception { + // The matcher supports embedded placeholders now, so these two are + // perfectly writable -- but the duplicate-shape check collapsed each whole + // segment to "{}", making them identical and refusing the contract. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.FileApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface FileApi {\n" + + " @GET(\"/{name}.json\")\n" + + " void json(@Path(\"name\") String name,\n" + + " OnComplete> callback);\n" + + " @GET(\"/{name}.xml\")\n" + + " void xml(@Path(\"name\") String name,\n" + + " OnComplete> callback);\n" + + "}\n"); + ProcessorContext ctx = runProcessor(compileSources(sources)); + assertFalse("no request satisfies both, so they are not duplicates: " + + ctx.getErrors(), ctx.hasErrors()); + } + + @Test + public void aRelativeTemplateStillMatchesTheRequest() throws Exception { + // A contract written without the leading slash resolves against a base URL + // ending in "/", so the CLIENT requests /notes. The server split the + // template to one segment while the incoming path splits to two, so the + // route could never match -- a contract that works as a client and answers + // nothing as a server. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.RelApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface RelApi {\n" + + " @GET(\"notes\")\n" + + " void all(OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.RelApiServer"); + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + return "ok"; + } + }); + Class dispatcherClass = loader.loadClass("com.example.RelApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + assertNotNull("the route the client requests must be the one the server answers", + dispatch.invoke(dispatcher, "GET", "/notes", null, null)); + loader.close(); + } + + @Test + public void theDispatcherRefusesAMalformedEscape() throws Exception { + // The same rule as the router's, in the other generator -- and the + // decoder here had a second hole besides: Integer.parseInt(_, 16) accepts + // a sign, so "%+1" decoded to the byte 1. Two more spellings of a value + // the client never wrote. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.NoteApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface NoteApi {\n" + + " @GET(\"/notes/{id}\")\n" + + " void byId(@Path(\"id\") String id, OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.NoteApiServer"); + final Object[] seen = new Object[1]; + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + seen[0] = args[0]; + return "ok"; + } + }); + Class dispatcherClass = loader.loadClass("com.example.NoteApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + + dispatch.invoke(dispatcher, "GET", "/notes/%41", null, null); + assertEquals("a well-formed escape still decodes", "A", seen[0]); + + seen[0] = null; + try { + dispatch.invoke(dispatcher, "GET", "/notes/%ZZ", null, null); + fail("a non-hex escape should be refused, not passed through as text"); + } catch (java.lang.reflect.InvocationTargetException expected) { + assertTrue(String.valueOf(expected.getCause()), + expected.getCause() instanceof IllegalArgumentException); + } + assertNull("the handler must not have run", seen[0]); + + try { + dispatch.invoke(dispatcher, "GET", "/notes/%+1", null, null); + fail("a signed hex pair is not a hex pair"); + } catch (java.lang.reflect.InvocationTargetException expected) { + assertTrue(String.valueOf(expected.getCause()), + expected.getCause() instanceof IllegalArgumentException); + } + } + + @Test + public void twoDynamicRoutesThatOverlapAreRefused() throws Exception { + // Different shapes, and /a/b/c satisfies both. Neither is more specific, so + // literal-first ordering cannot break the tie and dispatch answers with + // whichever it emitted first. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.AmbiguousApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface AmbiguousApi {\n" + + " @GET(\"/a/{x}/c\")\n" + + " void one(@Path(\"x\") String x, OnComplete> callback);\n" + + " @GET(\"/a/b/{y}\")\n" + + " void two(@Path(\"y\") String y, OnComplete> callback);\n" + + "}\n"); + ProcessorContext ctx = runProcessor(compileSources(sources)); + assertTrue("two routes that both answer /a/b/c should not compile", ctx.hasErrors()); + } + + @Test + public void aJsonIntegerTooLargeForTheFieldIsRefused() throws Exception { + // The parser answers a Long for any JSON integer, and intValue() on + // 2147483648 is -2147483648: the handler used to be handed a different + // number from the one the client sent, silently. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.Counter", + "package com.example;\n" + + "public class Counter {\n" + + " public int count;\n" + + " public Counter() {}\n" + + "}\n"); + sources.put("com.example.CounterApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface CounterApi {\n" + + " @POST(\"/count\")\n" + + " void put(@Body Counter c, OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class codec = loader.loadClass("com.example.CounterJson"); + Method fromMap = codec.getMethod("fromMap", Map.class); + + Map inRange = new java.util.LinkedHashMap(); + inRange.put("count", Long.valueOf(7)); + Object decoded = fromMap.invoke(null, inRange); + assertEquals(7, loader.loadClass("com.example.Counter").getField("count").get(decoded)); + + Map tooLarge = new java.util.LinkedHashMap(); + tooLarge.put("count", Long.valueOf(2147483648L)); + try { + fromMap.invoke(null, tooLarge); + fail("a value that does not fit the field should not be narrowed into it"); + } catch (java.lang.reflect.InvocationTargetException expected) { + assertTrue(String.valueOf(expected.getCause()), + expected.getCause() instanceof IllegalArgumentException); + } + } + + @Test + public void aDtoCarriesTheFieldsItInherits() throws Exception { + // AnnotatedClass.getFields() reads one class file, so the base's fields were + // invisible to the codec: a Cat went over the wire with no species at all, + // and the decoder left it null on the way back. + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.Animal", + "package com.example;\n" + + "public class Animal {\n" + + " public String species;\n" + + " public Animal() {}\n" + + "}\n"); + sources.put("com.example.Cat", + "package com.example;\n" + + "public class Cat extends Animal {\n" + + " public String name;\n" + + " public Cat() {}\n" + + "}\n"); + sources.put("com.example.CatApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface CatApi {\n" + + " @GET(\"/cat\")\n" + + " void get(OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class cat = loader.loadClass("com.example.Cat"); + Object instance = cat.newInstance(); + cat.getField("name").set(instance, "Tom"); + cat.getField("species").set(instance, "felis"); + + Class codec = loader.loadClass("com.example.CatJson"); + Method toMap = codec.getMethod("toMap", cat); + Map encoded = (Map) toMap.invoke(null, instance); + assertEquals("Tom", encoded.get("name")); + assertEquals("the inherited field is missing from the wire shape", + "felis", encoded.get("species")); + + // And back, so the loss is not merely one-directional. + Method fromMap = codec.getMethod("fromMap", Map.class); + Object decoded = fromMap.invoke(null, encoded); + assertEquals("felis", cat.getField("species").get(decoded)); + } + + @Test + public void aMalformedBooleanIsRefusedRatherThanTakenAsFalse() throws Exception { + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.FlagApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface FlagApi {\n" + + " @GET(\"/flag\")\n" + + " void flag(@Query(\"on\") boolean on,\n" + + " OnComplete> callback);\n" + + "}\n"); + File classes = compileSources(sources); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.FlagApiServer"); + Object handler = java.lang.reflect.Proxy.newProxyInstance(loader, + new Class[]{serverItf}, new java.lang.reflect.InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + return "on=" + args[0]; + } + }); + Class dispatcherClass = loader.loadClass("com.example.FlagApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, Map.class, Object.class); + + assertEquals("on=true", dispatch.invoke(dispatcher, "GET", "/flag?on=true", null, null)); + assertEquals("on=false", dispatch.invoke(dispatcher, "GET", "/flag?on=false", null, null)); + // "treu" used to arrive as an explicit false, so the handler ran on a value + // the client never sent and nothing anywhere said so. + try { + dispatch.invoke(dispatcher, "GET", "/flag?on=treu", null, null); + fail("a value that is not a boolean should not bind as false"); + } catch (java.lang.reflect.InvocationTargetException expected) { + assertTrue(String.valueOf(expected.getCause()), + expected.getCause() instanceof IllegalArgumentException); + } + } + + @Test + public void aCollectionOfCollectionsOfDtosIsRefused() throws Exception { + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.Tag", TAG_SOURCE); + sources.put("com.example.NestedApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface NestedApi {\n" + + " @GET(\"/nested\")\n" + + " void nested(OnComplete>>> callback);\n" + + "}\n"); + ProcessorContext ctx = runProcessor(compileSources(sources)); + // The codec reaches the outer elements only, so the Tags inside would have + // been written as their toString(). A build error beats wrong JSON. + assertTrue("a shape the codec cannot encode should not compile", ctx.hasErrors()); + } + + @Test + public void generatesServerInterfaceAndWorkingDispatcher() throws Exception { + File classes = compileApi(); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + + assertTrue("server interface was not emitted", + new File(classes, "com/example/GreeterApiServer.class").isFile()); + assertTrue("dispatcher was not emitted", + new File(classes, "com/example/GreeterApiDispatcher.class").isFile()); + + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.GreeterApiServer"); + + // The callback parameter must have become the return type, and the + // callback itself must be gone from the signature. + Method greet = serverItf.getMethod("greet", String.class, String.class); + assertEquals(String.class, greet.getReturnType()); + + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) throws Exception { + String n = m.getName(); + if ("greet".equals(n)) return "hello " + args[0] + "/loud=" + args[1]; + if ("echo".equals(n)) return "echoed:" + args[0]; + if ("whoami".equals(n)) return "user=" + args[0] + ",session=" + args[1]; + if ("addPet".equals(n)) { + Object pet = args[0]; + // round-trips the DTO straight back out + return pet; + } + if ("listPets".equals(n)) { + Class petClass = proxy.getClass().getClassLoader().loadClass("com.example.Pet"); + Object p1 = petClass.newInstance(); + petClass.getField("id").setLong(p1, 7L); + petClass.getField("name").set(p1, "Rex"); + java.util.List out = new java.util.ArrayList(); + out.add(p1); + return out; + } + return null; + } + }); + + Class dispatcherClass = loader.loadClass("com.example.GreeterApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + Method hasRoute = dispatcherClass.getMethod("hasRoute", String.class, String.class); + + assertEquals("hello Shai/loud=yes", + dispatch.invoke(dispatcher, "GET", "/greet/Shai?loud=yes", null, null)); + // Absent query parameter binds to null rather than failing the route. + assertEquals("hello Shai/loud=null", + dispatch.invoke(dispatcher, "GET", "/greet/Shai", null, null)); + // Percent-encoding in a path segment is decoded before binding. + assertEquals("hello Shai Almog/loud=null", + dispatch.invoke(dispatcher, "GET", "/greet/Shai%20Almog", null, null)); + // '+' is a space in a query value. + assertEquals("hello Shai/loud=a b", + dispatch.invoke(dispatcher, "GET", "/greet/Shai?loud=a+b", null, null)); + assertEquals("echoed:{\"a\":1}", + dispatch.invoke(dispatcher, "POST", "/echo", null, "{\"a\":1}")); + + // Headers bind case-insensitively; cookies come out of the Cookie header. + java.util.Map headers = new java.util.LinkedHashMap(); + headers.put("x-user", "shai"); + headers.put("Cookie", "theme=dark; session=abc123; other=x"); + assertEquals("user=shai,session=abc123", + dispatch.invoke(dispatcher, "GET", "/whoami", headers, null)); + assertEquals("user=null,session=null", + dispatch.invoke(dispatcher, "GET", "/whoami", null, null)); + + // A DTO body is decoded from the request Map into the typed parameter, and + // a DTO result is encoded back to a Map. + java.util.Map petIn = new java.util.LinkedHashMap(); + petIn.put("id", Long.valueOf(42)); // the JSON reader hands integers back as Long + petIn.put("name", "Fido"); + petIn.put("good", Boolean.TRUE); + petIn.put("weight", Double.valueOf(12.5)); + Object out = dispatch.invoke(dispatcher, "POST", "/pet", null, petIn); + assertTrue("a DTO result must come back as a Map", out instanceof java.util.Map); + java.util.Map petOut = (java.util.Map) out; + assertEquals(Long.valueOf(42), petOut.get("id")); + assertEquals("Fido", petOut.get("name")); + assertEquals(Boolean.TRUE, petOut.get("good")); + assertEquals(Double.valueOf(12.5), petOut.get("weight")); + + // A List result is encoded element by element. + Object listOut = dispatch.invoke(dispatcher, "GET", "/pets", null, null); + assertTrue(listOut instanceof java.util.List); + java.util.Map first = (java.util.Map) ((java.util.List) listOut).get(0); + assertEquals("Rex", first.get("name")); + assertEquals(Long.valueOf(7), first.get("id")); + + // Route matching is by verb AND path, and hasRoute is what separates + // "no such route" from "the handler returned null". + assertTrue((Boolean) hasRoute.invoke(dispatcher, "GET", "/greet/Shai")); + assertTrue(!(Boolean) hasRoute.invoke(dispatcher, "GET", "/nope")); + assertTrue(!(Boolean) hasRoute.invoke(dispatcher, "POST", "/greet/Shai")); + assertNull(dispatch.invoke(dispatcher, "GET", "/nope", null, null)); + loader.close(); + } + + /** + * A DTO with a public final field cannot be round-tripped, so it is refused. + * + * The encoder writes every public field and the decoder assigns them after a + * no-argument construction, so a final one goes out and silently does not come + * back: the handler sees the initializer and the client's value is gone, with + * nothing failing to say so. A contract that cannot be honoured should not + * compile. + */ + @Test + public void refusesADtoWithAFinalField() throws Exception { + File classes = tmp.newFolder(); + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.Frozen", + "package com.example;\n" + + "public class Frozen {\n" + + " public final String label = \"set at construction\";\n" + + " public String mutable;\n" + + " public Frozen() {}\n" + + "}\n"); + sources.put("com.example.FrozenApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface FrozenApi {\n" + + " @POST(\"/frozen\")\n" + + " void send(@Body Frozen f, OnComplete> callback);\n" + + "}\n"); + JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); + assertTrue("a public final DTO field must fail the build, not be dropped in transit", + runProcessor(classes).hasErrors()); + } + + /** + * A DTO's collection FIELD arrives as its declared element type too. + * + * The body-parameter case was fixed first; this is the same defect one level in, + * where the elements land in a field rather than an argument. A List + * full of Longs is a ClassCastException on the JVM at the first read, and on the + * translated target a Long read as an Integer with no complaint at all. + */ + @Test + public void convertsScalarElementsInADtoCollectionField() throws Exception { + File classes = compileApi(); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + URLClassLoader loader = new URLClassLoader( + new java.net.URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.GreeterApiServer"); + final Object[] received = new Object[1]; + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + if ("addPet".equals(m.getName())) { + received[0] = args[0]; + return args[0]; + } + return null; + } + }); + Class dispatcherClass = loader.loadClass("com.example.GreeterApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + + java.util.Map body = new java.util.LinkedHashMap(); + body.put("name", "Rex"); + // What the JSON reader really produces for [3, 4]. + body.put("weights", java.util.Arrays.asList(Long.valueOf(3), Long.valueOf(4))); + dispatch.invoke(dispatcher, "POST", "/pet", null, body); + + assertNotNull("the DTO never reached the handler", received[0]); + java.util.List weights = (java.util.List) + received[0].getClass().getField("weights").get(received[0]); + assertNotNull("the collection field was not decoded", weights); + assertEquals("an Integer element must not still be a Long", + Integer.class, weights.get(0).getClass()); + assertEquals(Integer.valueOf(3), weights.get(0)); + loader.close(); + } + + /** + * A collection body arrives as its DECLARED element type, not the parser's. + * + * The JSON reader produces Long for every integer, so a `List` handed + * over raw is a list of Longs wearing a List label. The JVM reveals + * that as a ClassCastException the first time the handler reads an element; + * ParparVM's CHECKCAST is unchecked, so there it reads an Integer's fields out + * of a Long and keeps going. Asserting on the element's runtime class is the + * only way to see the difference. + */ + @Test + public void convertsScalarElementsInACollectionBody() throws Exception { + File classes = compileApi(); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + URLClassLoader loader = new URLClassLoader( + new java.net.URL[]{ classes.toURI().toURL() }, getClass().getClassLoader()); + Class serverInterface = loader.loadClass("com.example.GreeterApiServer"); + final Object[] received = new Object[2]; + Object impl = Proxy.newProxyInstance(loader, new Class[]{ serverInterface }, + new InvocationHandler() { + public Object invoke(Object proxy, Method method, Object[] args) { + if ("weights".equals(method.getName())) { + received[0] = args[0]; + } else if ("labels".equals(method.getName())) { + received[1] = args[0]; + } + return "ok"; + } + }); + Class dispatcherClass = loader.loadClass("com.example.GreeterApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverInterface).newInstance(impl); + Method dispatch = dispatcherClass.getMethod("dispatch", String.class, String.class, + java.util.Map.class, Object.class); + + dispatch.invoke(dispatcher, "POST", "/weights", null, + java.util.Arrays.asList(Long.valueOf(3), Long.valueOf(4))); + java.util.List weights = (java.util.List) received[0]; + assertNotNull("the list body did not reach the handler", weights); + assertEquals("an Integer element must not still be a Long", + Integer.class, weights.get(0).getClass()); + assertEquals(Integer.valueOf(3), weights.get(0)); + + dispatch.invoke(dispatcher, "POST", "/labels", null, + java.util.Arrays.asList("a", "b")); + assertTrue("a Set body must arrive as a Set", received[1] instanceof java.util.Set); + assertEquals(2, ((java.util.Set) received[1]).size()); + loader.close(); + } + + /** + * The request body's SHAPE is the client's choice, so nothing the dispatcher + * does with it may rest on a cast. + * + * This is not a style point. ParparVM's CHECKCAST is unchecked by default (see + * CLAUDE.md), so `(Map)body` over a String does not throw on a translated + * server -- it reads a String's header as a Map's, and the process dies taking + * every in-flight connection with it. A four-byte body once did exactly that. + * The JVM only reveals it as a ClassCastException, which is why this asserts on + * the MESSAGE: an IllegalArgumentException naming the expected shape is a 400, + * and a ClassCastException is a 500 here and a crash there. + */ + @Test + public void refusesABodyOfTheWrongShapeInsteadOfCastingIt() throws Exception { + File classes = compileApi(); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + Class serverItf = loader.loadClass("com.example.GreeterApiServer"); + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) { + return "addPet".equals(m.getName()) ? args[0] : null; + } + }); + Class dispatcherClass = loader.loadClass("com.example.GreeterApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + + // A route declaring a DTO, handed a string, a number and an array. + Object[] wrongShapes = new Object[]{"not an object", Long.valueOf(42), + new java.util.ArrayList()}; + for (Object wrong : wrongShapes) { + try { + dispatch.invoke(dispatcher, "POST", "/pet", null, wrong); + fail("a " + wrong.getClass().getSimpleName() + + " body must be rejected, not cast to a Map"); + } catch (java.lang.reflect.InvocationTargetException err) { + Throwable cause = err.getCause(); + assertTrue("expected a 400-shaped rejection, got " + cause, + cause instanceof IllegalArgumentException); + assertTrue("the message must say what was expected: " + cause.getMessage(), + cause.getMessage().indexOf("JSON object") >= 0); + } + } + + // Null stays null: an absent body is not a malformed one. + assertNull(dispatch.invoke(dispatcher, "POST", "/pet", null, null)); + loader.close(); + } + + /** + * A DTO-typed collection field has to be converted element by element in BOTH + * directions. Handing the handler the decoded Maps typed as Tags is a lie the + * JVM catches at the first field read and a translated binary does not catch at + * all; writing the Tags back without converting them serialises toString(). + */ + @Test + public void roundTripsACollectionOfNestedDtos() throws Exception { + File classes = compileApi(); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + + URLClassLoader loader = new URLClassLoader( + new URL[]{classes.toURI().toURL(), testClassesDir().toURI().toURL()}, + getClass().getClassLoader()); + final Class tagClass = loader.loadClass("com.example.Tag"); + Class serverItf = loader.loadClass("com.example.GreeterApiServer"); + // The handler READS a typed field off every element, which is the operation + // a List of Maps typed as Tags fails at. + Object handler = Proxy.newProxyInstance(loader, new Class[]{serverItf}, + new InvocationHandler() { + public Object invoke(Object proxy, Method m, Object[] args) throws Exception { + if (!"addPet".equals(m.getName())) { + return null; + } + Object pet = args[0]; + java.util.List tags = (java.util.List) pet.getClass() + .getField("tags").get(pet); + double total = 0; + for (int i = 0; i < tags.size(); i++) { + Object tag = tags.get(i); + if (tag == null) { + continue; // an element of the wrong shape decodes to null + } + assertTrue("element " + i + " is a " + tag.getClass().getName() + + ", not a Tag", tagClass.isInstance(tag)); + total += tagClass.getField("weight").getInt(tag); + } + pet.getClass().getField("weight").setDouble(pet, total); + return pet; + } + }); + Class dispatcherClass = loader.loadClass("com.example.GreeterApiDispatcher"); + Object dispatcher = dispatcherClass.getConstructor(serverItf).newInstance(handler); + Method dispatch = dispatcherClass.getMethod("dispatch", + String.class, String.class, java.util.Map.class, Object.class); + + java.util.Map friendly = new java.util.LinkedHashMap(); + friendly.put("label", "friendly"); + friendly.put("weight", Long.valueOf(3)); + java.util.Map loud = new java.util.LinkedHashMap(); + loud.put("label", "loud"); + loud.put("weight", Long.valueOf(1)); + java.util.List tagMaps = new java.util.ArrayList(); + tagMaps.add(friendly); + tagMaps.add(loud); + java.util.Map petIn = new java.util.LinkedHashMap(); + petIn.put("name", "Rex"); + petIn.put("tags", tagMaps); + + java.util.Map petOut = (java.util.Map) dispatch.invoke(dispatcher, "POST", "/pet", + null, petIn); + assertEquals(Double.valueOf(4), petOut.get("weight")); + java.util.List tagsOut = (java.util.List) petOut.get("tags"); + assertTrue("nested DTOs must be written back as Maps, not as objects", + tagsOut.get(0) instanceof java.util.Map); + assertEquals("friendly", ((java.util.Map) tagsOut.get(0)).get("label")); + + // An element of the wrong shape is REFUSED. This assertion used to expect + // null, which was itself an improvement on handing the handler a Map + // wearing a Tag's type -- but null is a value the client can legitimately + // send, so substituting it for a mistake made the two indistinguishable + // and let the handler act on a collection with a hole in it. Throwing is + // what lets the transport answer 400, which is what the request deserves. + java.util.List mixed = new java.util.ArrayList(); + mixed.add("not an object"); + java.util.Map petMixed = new java.util.LinkedHashMap(); + petMixed.put("tags", mixed); + try { + dispatch.invoke(dispatcher, "POST", "/pet", null, petMixed); + fail("a scalar where a Tag was declared must be refused, not nulled"); + } catch (java.lang.reflect.InvocationTargetException expected) { + assertTrue(String.valueOf(expected.getCause()), + expected.getCause() instanceof IllegalArgumentException); + } + // A genuine JSON null element still passes through as null. + java.util.List withNull = new java.util.ArrayList(); + withNull.add(null); + java.util.Map petNull = new java.util.LinkedHashMap(); + petNull.put("tags", withNull); + java.util.Map nullOut = (java.util.Map) dispatch.invoke(dispatcher, "POST", "/pet", + null, petNull); + assertNull(((java.util.List) nullOut.get("tags")).get(0)); + loader.close(); + } + + @Test + public void refusesAParameterItCannotBind() throws Exception { + File classes = tmp.newFolder("classes-unbindable"); + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.HeaderApi", + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface HeaderApi {\n" + + " @GET(\"/thing\")\n" + + " void thing(String noAnnotation,\n" + + " OnComplete> callback);\n" + + "}\n"); + JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); + ProcessorContext ctx = runProcessor(classes); + assertTrue("a parameter with no binding annotation must fail the build, not bind to null", + ctx.hasErrors()); + } + + /** + * A @Path that names no placeholder is a typo, and it used to bind null -- or 0 + * for a primitive -- while the route still matched, so the handler ran with the + * wrong identifier and the build said nothing. + */ + @Test + public void refusesAPathBindingThatMatchesNoPlaceholder() throws Exception { + assertTrue("a @Path naming no placeholder must fail the build", + processApi("TypoApi", + " @GET(\"/users/{id}\")\n" + + " void user(@Path(\"userId\") String id,\n" + + " OnComplete> callback);\n").hasErrors()); + } + + /** + * Two routes of one verb and shape compile to the same predicate, and dispatch + * returns on the first match -- so the second can never be reached however it is + * called. The placeholder NAMES differ; the router never sees them. + */ + @Test + public void refusesTwoRoutesOfTheSameShape() throws Exception { + assertTrue("an unreachable duplicate route must fail the build", + processApi("AmbiguousApi", + " @GET(\"/pets/{id}\")\n" + + " void byId(@Path(\"id\") String id,\n" + + " OnComplete> callback);\n" + + " @GET(\"/pets/{name}\")\n" + + " void byName(@Path(\"name\") String name,\n" + + " OnComplete> callback);\n").hasErrors()); + } + + /** + * Collection belongs with List and Set. The parser answers an ArrayList for + * every JSON array, so a Collection that is not recognised as a collection + * SHAPE falls through to a guarded cast, which erases -- leaving a collection + * of Map under a Collection declaration, which throws on first use. + */ + @Test + public void decodesACollectionBodyLikeAListOne() throws Exception { + assertNoErrors(processApi("CollectionApi", + " @POST(\"/notes\")\n" + + " void add(@Body java.util.Collection notes,\n" + + " OnComplete> callback);\n")); + } + + /** + * A Map's values are handed over as the parser built them, and nothing walks + * them applying the declared type the way collection elements are walked. So + * Map is a map of Long at runtime and the handler's first + * read as an Integer throws, from a contract that processed cleanly. + */ + @Test + public void refusesAMapOfATypeTheParserDoesNotProduce() throws Exception { + assertTrue("a map of Integer must fail the build", + processApi("MapApi", + " @POST(\"/counts\")\n" + + " void put(@Body java.util.Map counts,\n" + + " OnComplete> callback);\n").hasErrors()); + } + + /** A map of what the parser DOES produce still works. */ + @Test + public void allowsAMapOfTheTypesTheParserProduces() throws Exception { + assertNoErrors(processApi("MapOkApi", + " @POST(\"/counts\")\n" + + " void put(@Body java.util.Map counts,\n" + + " OnComplete> callback);\n")); + } + + /** + * A placeholder nothing binds. The client substitutes the placeholder's own + * NAME, so it asks for /users/id literally, while the server matches any + * value there and hands it to nobody -- two halves agreeing on a route whose + * variable cannot be supplied or read. + */ + @Test + public void refusesAPlaceholderNothingBinds() throws Exception { + assertTrue("a placeholder with no @Path must fail the build", + processApi("UnboundApi", + " @GET(\"/users/{id}\")\n" + + " void user(OnComplete> callback);\n").hasErrors()); + } + + /** + * A literal beside a placeholder is NOT ambiguous: the dispatcher emits every + * route without a placeholder before every route with one, so /users/me takes + * its own path and every other value falls through to {id}. This is the most + * ordinary pair there is, and refusing it left no way to write it. + */ + @Test + public void allowsALiteralBesideAPlaceholder() throws Exception { + assertNoErrors(processApi("LiteralApi", + " @GET(\"/users/me\")\n" + + " void me(OnComplete> callback);\n" + + " @GET(\"/users/{id}\")\n" + + " void byId(@Path(\"id\") String id,\n" + + " OnComplete> callback);\n")); + } + + /** Two routes of the same shape but DIFFERENT verbs are not ambiguous. */ + @Test + public void allowsTheSameShapeUnderDifferentVerbs() throws Exception { + assertNoErrors(processApi("VerbsApi", + " @GET(\"/pets/{id}\")\n" + + " void read(@Path(\"id\") String id,\n" + + " OnComplete> callback);\n" + + " @DELETE(\"/pets/{id}\")\n" + + " void remove(@Path(\"id\") String id,\n" + + " OnComplete> callback);\n")); + } + + /** Compiles one throwaway contract and runs the processor over it. */ + private ProcessorContext processApi(String name, String methods) throws Exception { + File classes = tmp.newFolder(); + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example." + name, + "package com.example;\n" + + "import com.codename1.annotations.rest.*;\n" + + "import com.codename1.io.rest.Response;\n" + + "import com.codename1.util.OnComplete;\n" + + "@RestClient\n" + + "public interface " + name + " {\n" + + methods + + "}\n"); + JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); + return runProcessor(classes); + } + + @Test + public void generatesNothingWhenTheServerHalfIsOff() throws Exception { + System.clearProperty("cn1.restServer"); + File classes = compileApi(); + ProcessorContext ctx = runProcessor(classes); + assertNoErrors(ctx); + assertTrue("the server half must be opt-in so existing app builds do not grow", + !new File(classes, "com/example/GreeterApiDispatcher.class").isFile()); + } + + /** Compiles an arbitrary set of sources, for the cases the shared fixture cannot express. */ + private File compileSources(Map sources) throws Exception { + File classes = tmp.newFolder(); + JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); + return classes; + } + + private File compileApi() throws Exception { + File classes = tmp.newFolder(); + Map sources = new java.util.LinkedHashMap(); + sources.put("com.example.Tag", TAG_SOURCE); + sources.put("com.example.Pet", DTO_SOURCE); + sources.put("com.example.GreeterApi", API_SOURCE); + JavaSourceCompiler.compile(sources, classes, Arrays.asList(testClassesDir())); + return classes; + } + + private void assertNoErrors(ProcessorContext ctx) { + if (ctx.hasErrors()) { + StringBuilder sb = new StringBuilder("processor reported errors:\n"); + for (ProcessorContext.ProcessingError e : ctx.getErrors()) sb.append(' ').append(e).append('\n'); + fail(sb.toString()); + } + } + + private ProcessorContext runProcessor(File classesDir) throws Exception { + Map index = ClassScanner.scan(classesDir); + RestServerAnnotationProcessor proc = new RestServerAnnotationProcessor(); + ProcessorContext ctx = new ProcessorContext(classesDir, tmp.newFolder(), + index, new SystemStreamLog()); + proc.start(ctx); + for (AnnotatedClass cls : index.values()) { + if (!cls.getClassAnnotations().isEmpty()) proc.processClass(cls, ctx); + } + proc.finish(ctx); + return ctx; + } + + private static File testClassesDir() throws Exception { + URL url = RestServerAnnotationProcessorTest.class.getProtectionDomain() + .getCodeSource().getLocation(); + return new File(url.toURI()); + } +} diff --git a/maven/integration-tests/cn1app-archetype-test.sh b/maven/integration-tests/cn1app-archetype-test.sh index fd47756e620..2c672873a2c 100644 --- a/maven/integration-tests/cn1app-archetype-test.sh +++ b/maven/integration-tests/cn1app-archetype-test.sh @@ -33,4 +33,35 @@ if [ -d /Applications/Xcode.app ]; then fi if [ -d $HOME/Library/Android/sdk ]; then "mvn" "package" "-DskipTests" "-Dcodename1.platform=android" "-Dcodename1.buildTarget=android-source" -Dopen=false -fi \ No newline at end of file +fi +# The backend module, which no other step here reaches: build.sh above builds the +# client, and the two platform builds are behind their own SDK checks. +# +# It earns its place. The generated server is a @RestController with no main of its +# own -- the router and the entry point are generated from it during +# process-classes -- so "does the template still compile" and "does the generator +# still produce an entry point" are two different questions and this asks both. The +# reason it is here at all is that the template once shipped with its copyright +# header missing the closing "*/", which put the package declaration and every +# import inside a comment; it was found by hand, and nothing in CI would have said +# a word. +mvn -pl backend -Dcodename1.platform=backend process-classes + +MAIN_CLASS_FILE="backend/target/classes/META-INF/cn1-backend-main" +if [ ! -f "$MAIN_CLASS_FILE" ]; then + echo "the backend module did not record a generated entry point" >&2 + exit 1 +fi +GENERATED_MAIN="$(cat "$MAIN_CLASS_FILE")" +echo "backend entry point: $GENERATED_MAIN" +if [ ! -f "backend/target/classes/$(echo "$GENERATED_MAIN" | tr '.' '/').class" ]; then + echo "the recorded entry point $GENERATED_MAIN was not compiled" >&2 + exit 1 +fi +# The router lands beside the entry point, whatever package the archetype was told +# to use -- derived rather than assumed, since `package` defaults to the groupId. +ROUTER_DIR="$(dirname "$(echo "$GENERATED_MAIN" | tr '.' '/')")" +if [ ! -f "backend/target/classes/$ROUTER_DIR/ApiRouter.class" ]; then + echo "no router was generated for the @RestController" >&2 + exit 1 +fi diff --git a/maven/integration-tests/validate_initializr_pom_coordinates.py b/maven/integration-tests/validate_initializr_pom_coordinates.py index e6e7ea4261f..30233a562ab 100644 --- a/maven/integration-tests/validate_initializr_pom_coordinates.py +++ b/maven/integration-tests/validate_initializr_pom_coordinates.py @@ -36,6 +36,12 @@ ROOT_ARTIFACT_ID = "myappname" ROOT_VERSION = "1.0-SNAPSHOT" PLATFORMS = ("android", "ios", "javase", "javascript", "linux", "win") +# The backend module is shaped like a platform module and validated like one, but it +# must NOT depend on the generated common module: common is compiled against +# codenameone-core and a server has no display. Requiring it here would enforce the +# mistake the module's own comment warns against, so it is validated separately with +# that one check turned off. +BACKEND = "backend" def fail(message): @@ -84,7 +90,7 @@ def validate_root_pom(archive): reject_initializr_coordinates(data, "pom.xml") -def validate_platform_pom(archive, platform): +def validate_platform_pom(archive, platform, require_common_dependency=True): path = platform + "/pom.xml" data, project = read_pom(archive, path) parent = project.find("m:parent", NS) @@ -115,8 +121,10 @@ def validate_platform_pom(archive, platform): "${project.groupId}", path + "/common-dependency", "groupId") require_equal(direct_text(dependency, "version", path + "/common-dependency"), "${project.version}", path + "/common-dependency", "version") - if not common_dependency_found: + if require_common_dependency and not common_dependency_found: fail(path + " does not depend on ${project.groupId}:${cn1app.name}-common:${project.version}") + if not require_common_dependency and common_dependency_found: + fail(path + " must not depend on the generated common module") reject_initializr_coordinates(data, path) @@ -141,7 +149,8 @@ def main(): # common/pom.xml is deliberately not stored in common.zip: GeneratorModel # injects the selected template's common POM after reading this artifact. # The generated common POM is covered by its runtime guard and matrix tests. - expected_poms = {"pom.xml"} | {platform + "/pom.xml" for platform in PLATFORMS} + expected_poms = ({"pom.xml"} | {platform + "/pom.xml" for platform in PLATFORMS} + | {BACKEND + "/pom.xml"}) if embedded_poms != expected_poms: fail("Initializr artifact POM set differs from the validated platform set: found " + repr(sorted(embedded_poms)) + ", expected " + repr(sorted(expected_poms))) @@ -149,6 +158,7 @@ def main(): validate_root_pom(archive) for platform in PLATFORMS: validate_platform_pom(archive, platform) + validate_platform_pom(archive, BACKEND, require_common_dependency=False) print("Initializr embedded POM coordinates are consistent across all platform modules.") diff --git a/maven/pom.xml b/maven/pom.xml index fa8ff96e09e..1ff7f295d92 100644 --- a/maven/pom.xml +++ b/maven/pom.xml @@ -90,6 +90,8 @@ svg-transcoder lottie-transcoder sqlite-jdbc + + backend javase javase-svg + + 4.0.0 + + com.codenameone.backend + cn1-backend-contract + 8.0-SNAPSHOT + jar + Codename One backend contract + + + 8.0-SNAPSHOT + 1.8 + 1.8 + UTF-8 + + + + + com.codenameone + codenameone-core + ${cn1.version} + provided + + + + + ${project.basedir} + + + com.codenameone + codenameone-maven-plugin + ${cn1.version} + + + generate-server-half + process-classes + + process-annotations + + + + + + + diff --git a/vm/backend/demo/bench/com/demo/Bench.java b/vm/backend/demo/bench/com/demo/Bench.java new file mode 100644 index 00000000000..4f1a3e845bf --- /dev/null +++ b/vm/backend/demo/bench/com/demo/Bench.java @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.LinkedHashMap; +import java.util.Map; + +import com.codename1.backend.HttpServer; +import com.codename1.backend.Json; +import com.codename1.backend.Signals; + +/** + * The Codename One half of the Go comparison. + * + * Two routes, deliberately the two TechEmpower framework-benchmark shapes, so the + * numbers here can be read against published ones as well as against the Go + * server beside them: + * + * /plaintext text/plain, a fixed 13-byte body + * /json application/json, a small object serialised per request + * + * The JSON is built per request rather than served from a constant, because + * serialisation is part of what is being compared; the plaintext route is the one + * that measures the HTTP path with nothing else in it. + * + * bench-server.go is line-for-line the same two handlers on net/http. Anything + * this file does that that one does not -- or the other way round -- is a + * difference in the measurement, not in the runtimes, so keep them matched. + */ +public class Bench { + private static final byte[] PLAINTEXT = bytes("Hello, World!"); + private static final int RESPONSE_MODE = envInt("BENCH_REUSE_RESPONSE", 1); + + /** + * 0 = build a LinkedHashMap per request (what a hand-written handler does), + * 1 = reuse one map (isolates construction from serialising), + * 2 = write the fields directly (what the annotation processor now emits). + */ + private static final int JSON_MODE = envInt("BENCH_JSON_MODE", 0); + + /** Reused; the object is immutable and the writer holds no state. */ + private static final com.codename1.backend.Json.Writable MESSAGE_WRITABLE = + new com.codename1.backend.Json.Writable() { + public void writeTo(com.codename1.backend.ByteSink out) { + out.put('{'); + out.putAscii("\"message\":"); + com.codename1.backend.Json.writeString("Hello, World!", out); + out.put('}'); + } + }; + + private static final Map HOISTED = new LinkedHashMap(); + static { + HOISTED.put("message", "Hello, World!"); + } + + public static void main(String[] args) throws Exception { + Signals.installShutdownHandler(); + int port = envInt("PORT", 8080); + int workers = envInt("WORKERS", 16); + int backlog = envInt("BACKLOG", 1024); + + // DIAGNOSTIC (BENCH_IDLE_THREADS=N): park N Java threads that do nothing. + // + // The collector stops and scans threads ONE AT A TIME, and it rebuilds its + // virtual-thread snapshot inside that per-thread loop -- so the more Java + // threads exist, the longer a snapshot stays live while OTHER threads are + // still running and still free virtual threads. Idle threads therefore + // widen a race they take no part in. This reproduces that without the + // worker pool, which used to supply the threads and no longer exists in + // virtual-thread mode. + int idle = envInt("BENCH_IDLE_THREADS", 0); + for(int iter = 0 ; iter < idle ; iter++) { + Thread parked = new Thread(new Runnable() { + public void run() { + while(true) { + try { + Thread.sleep(3600000); + } catch (InterruptedException err) { + return; + } + } + } + }); + parked.setDaemon(true); + parked.start(); + } + + final HttpServer server = HttpServer.start(null, port, backlog, workers, + new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + String target = request.getTarget(); + // startsWith, so BENCH_VARY_TARGETS can drive a DISTINCT target per + // request (/plaintext?u=N) and still be served 200 + keep-alive. + // Matching exactly sends those to the 404 path, which replies + // Connection: close -- and a red-team run of the target cache then + // measures connection teardown rather than the cache: 292 requests + // in 5.1s with 758,381 write errors. Prefix matching keeps the + // adversarial case on the same code path as the normal one. + if(target.startsWith("/plaintext")) { + // BENCH_REUSE_RESPONSE=1: hand back one shared Response + // instead of building one per request. + // + // Not a shippable handler -- it measures a CEILING. Response + // is the only per-request allocation left on this route (88 + // bytes), and the server only ever READS it, so sharing one + // is safe here and answers what pooling would be worth before + // any public API is changed to allow it. + // No per-request Response: the connection's own is re-pointed. + // BENCH_REUSE_RESPONSE=0 restores the allocating path, which is + // what the comparison measures against. + // 0 = allocate per request + // 1 = pooled, re-pointed via respond() (nine field writes) + // 2 = pooled but PRE-SET, returned untouched + // + // Mode 2 exists to separate two things mode 1 conflates: the + // saved allocation, and the cost of writing the fields into a + // connection-cold object instead of a bump-allocated one that + // is still warm in cache. Only valid because this route always + // answers with the same status, type and body. + if(RESPONSE_MODE == 2) { + HttpServer.Response r = request.presetResponse(); + if(r != null) { + return r; + } + } + if(RESPONSE_MODE == 1) { + return request.respond(200, "text/plain", PLAINTEXT); + } + return new HttpServer.Response(200, "text/plain", PLAINTEXT); + } + if("/json".equals(target)) { + // DIAGNOSTIC SPLIT (BENCH_JSON_HOIST=1): reuse one map instead + // of building it per request. + // + // Not a shippable handler -- a real one has different values + // each time -- but it separates the two costs this route pays. + // Go encodes a STRUCT with a cached per-type encoder; we build + // a LinkedHashMap, hash a key, insert, then walk it. Those are + // not the same work, so before concluding "our JSON serialiser + // is slow" it is worth knowing how much of the gap is the + // container rather than the serialising. + // + // Recovers most of the gap -> the fix is a struct-shaped API in + // plain Java. Recovers little -> the cost really is in the byte + // writer, and porting that to C is justified. + // + // ANSWERED, and it is the container. Two pinned cores, 64 + // connections, interleaved with rotating arm order, n=2: + // + // generated DTO (2) 595158 rps 2.619 us/req + // fasthttp 585906 rps 2.525 us/req + // hoisted map (1) 547423 rps 2.849 us/req + // map per request (0) 338167 rps 4.512 us/req + // + // Building the map costs 1.9 us of the 2.0 us that separated + // this route from Go -- hoisting it alone recovers most of that, + // and the struct-shaped writer recovers the rest and passes + // fasthttp. So the byte writer does NOT need porting to C: at + // mode 2 it is already serialising this object for less cpu than + // Go spends on the equivalent, and what looked like a serialiser + // gap was a LinkedHashMap allocated, hashed, inserted into and + // walked once per request. + // + // Mode 0 stays the default because it is the honest cost of a + // handler that hands back a Map, which is what an unannotated + // one does. An annotated DTO gets mode 2's shape from the + // processor without the author writing any of it. + if(JSON_MODE == 1) { + return HttpServer.Response.jsonValue(200, HOISTED); + } + if(JSON_MODE == 3) { + // Mode 2's writer on the connection's pooled Response, so the + // route allocates nothing at all. Mode 2 stays as it was so the + // cost of the Response itself remains measurable against it. + return request.respondJson(200, MESSAGE_WRITABLE); + } + if(JSON_MODE == 2) { + // What the annotation processor now emits for a DTO: no + // map, no key hashing, no walk, no instanceof per value -- + // the field name is a literal and the value takes the + // writer its static type selects. Hand-written here only + // because this benchmark handler is not annotated; the + // generated PetJson.toJson has exactly this shape. + return HttpServer.Response.jsonValue(200, MESSAGE_WRITABLE); + } + // Serialised per request, because the Go side encodes a struct + // per request. Returning a constant string here would compare + // our memcpy against their reflection. + Map out = new LinkedHashMap(); + out.put("message", "Hello, World!"); + // jsonValue, not json(Json.write(...)): the map is serialised + // straight into the connection's write buffer. The Go side + // encodes a struct per request, so this stays a real + // serialisation rather than a hoisted constant. + return HttpServer.Response.jsonValue(200, out); + } + return HttpServer.Response.text(404, "not found"); + } + }, null); + + System.out.println("bench listening on port " + server.getPort() + + " with " + workers + " workers"); + Signals.onShutdown(new Runnable() { + public void run() { + server.stop(2000); + // Signals ends the process; see PetServer for why not here. + } + }); + server.awaitTermination(); + } + + private static byte[] bytes(String value) { + try { + return value.getBytes("UTF-8"); + } catch (Exception err) { + return new byte[0]; + } + } + + private static int envInt(String name, int fallback) { + String value = System.getenv(name); + if(value == null || value.length() == 0) { + return fallback; + } + try { + return Integer.parseInt(value.trim()); + } catch (NumberFormatException err) { + return fallback; + } + } +} diff --git a/vm/backend/demo/common/com/demo/GreeterService.java b/vm/backend/demo/common/com/demo/GreeterService.java new file mode 100644 index 00000000000..88af8b66b80 --- /dev/null +++ b/vm/backend/demo/common/com/demo/GreeterService.java @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.codename1.backend.Crypto; +import com.codename1.backend.Db; +import com.codename1.backend.DbPool; +import com.codename1.backend.Jwt; +import com.codename1.backend.Web; + +/** + * The application code: implements the GENERATED server interface, whose + * signatures come from the shared GreeterApi contract. Persistence is real - rows + * go into SQLite through bound parameters, never string-concatenated SQL. + */ +public class GreeterService implements GreeterApiServer { + private static final long TOKEN_TTL_SECONDS = 3600; + + /** + * Exactly one of these is set. + * + * `pool` is the normal case: every call borrows a connection and gives it back, + * so two requests never share one. Holding a single borrowed connection for the + * life of the service -- which this used to do -- leaves the rest of the pool + * unused AND lets one request's statements land inside another's transaction, so + * a concurrent addPet could be rolled back by an unrelated addPets that failed. + * + * `shared` is the in-memory case, where pooling is not possible: each connection + * to ":memory:" would be its own empty database. SQLite serialises the one + * connection, so sharing it is correct there rather than merely convenient. + */ + private final DbPool pool; + private final Db shared; + private final byte[] signingSecret; + + public GreeterService(Db db) throws Exception { + this(db, Crypto.randomBytes(32)); + } + + public GreeterService(DbPool pool) throws Exception { + this(pool, Crypto.randomBytes(32)); + } + + public GreeterService(DbPool pool, byte[] signingSecret) throws Exception { + this.pool = pool; + this.shared = null; + this.signingSecret = signingSecret; + createSchema(); + } + + /** + * - `signingSecret`: at least 32 bytes. A real deployment reads this from its + * environment so tokens survive a restart and every instance agrees; the + * generated-per-process default is right for a demo and wrong for a fleet. + */ + public GreeterService(Db db, byte[] signingSecret) throws Exception { + this.pool = null; + this.shared = db; + this.signingSecret = signingSecret; + createSchema(); + } + + private void createSchema() throws Exception { + withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + db.execute("CREATE TABLE IF NOT EXISTS pet (" + + "id INTEGER PRIMARY KEY AUTOINCREMENT," + + "name TEXT NOT NULL," + + "species TEXT," + + "weight REAL," + + "good INTEGER," + + "photo BLOB)", null); + db.execute("CREATE TABLE IF NOT EXISTS account (" + + "username TEXT PRIMARY KEY," + + "password TEXT NOT NULL)", null); + // A demo account. Stored as a PBKDF2 verifier, never as the password. + if(db.query("SELECT username FROM account WHERE username = ?", + new Object[]{"shai"}).isEmpty()) { + db.execute("INSERT INTO account (username, password) VALUES (?, ?)", + new Object[]{"shai", Crypto.hashPassword("hunter2")}); + } + return null; + } + }); + } + + /** One borrowed connection for the duration of `body`, returned afterwards. */ + private Object withConnection(Db.Work body) throws Exception { + return pool == null ? body.run(shared) : pool.withConnection(body); + } + + /** As {@link #withConnection}, with the work wrapped in a transaction. */ + private Object inTransaction(Db.Work body) throws Exception { + return pool == null ? shared.transaction(body) : pool.inTransaction(body); + } + + public String login(Credentials credentials) throws Exception { + if(credentials == null || credentials.username == null) { + throw new IllegalArgumentException("username and password are required"); + } + final String username = credentials.username; + List rows = (List) withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + return db.query("SELECT password FROM account WHERE username = ?", + new Object[]{username}); + } + }); + // The same rejection for an unknown user and a wrong password: telling them + // apart turns the login endpoint into a list of valid usernames. + String stored = rows.isEmpty() ? null : str(((Map)rows.get(0)).get("password")); + if(!Crypto.verifyPassword(credentials.password, stored)) { + throw new SecurityException("bad credentials"); + } + Map claims = new java.util.LinkedHashMap(); + claims.put("sub", credentials.username); + return Jwt.issue(claims, signingSecret, TOKEN_TTL_SECONDS); + } + + /** The subject of a valid token, or a SecurityException. */ + private String requireCaller(String authorization) throws Exception { + String token = Jwt.bearer(authorization); + if(token == null) { + throw new SecurityException("a bearer token is required"); + } + try { + Map claims = Jwt.verify(token, signingSecret); + return str(claims.get("sub")); + } catch (Exception err) { + throw new SecurityException("invalid token"); + } + } + + public String greet(String name, String loud) throws Exception { + String greeting = "hello " + name; + return "yes".equals(loud) ? greeting.toUpperCase() : greeting; + } + + /** + * Hands the decoded DTO back, with its weight replaced by the total weight of + * its tags. Deliberately does not touch the database: what this proves is that + * the generated codec reads and writes the same shape, nested collections + * included. + * + * The tag loop is the point of the route. It reads a TYPED field off every + * element, which is what a List whose elements are decoded Maps typed as Tags + * fails at -- on the JVM with a ClassCastException, and on the native target + * by reading a Map's header as a Tag's, which is not survivable. + */ + public Pet echo(Pet pet) throws Exception { + if(pet == null) { + throw new IllegalArgumentException("a pet is required"); + } + if(pet.tags != null) { + int total = 0; + for(int iter = 0 ; iter < pet.tags.size() ; iter++) { + Tag tag = pet.tags.get(iter); + if(tag != null) { + total += tag.weight; + } + } + pet.weight = total; + } + return pet; + } + + public String whoami(String user, String session) throws Exception { + return "user=" + user + ",session=" + session; + } + + public Pet addPet(Pet pet) throws Exception { + if(pet == null || pet.name == null || pet.name.length() == 0) { + throw new IllegalArgumentException("a pet needs a name"); + } + final Pet inserting = pet; + // The insert and lastInsertId have to run on ONE connection: the id belongs + // to the connection that did the insert, so reading it from another is a + // different row or none at all. + pet.id = ((Long) withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + db.execute("INSERT INTO pet (name, species, weight, good) VALUES (?, ?, ?, ?)", + new Object[]{inserting.name, inserting.species, + new Double(inserting.weight), + Boolean.valueOf(inserting.good)}); + return new Long(db.lastInsertId()); + } + })).longValue(); + return pet; + } + + public Pet getPet(long id) throws Exception { + final long wanted = id; + List rows = (List) withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + return db.query("SELECT id, name, species, weight, good FROM pet WHERE id = ?", + new Object[]{new Long(wanted)}); + } + }); + if(rows.isEmpty()) { + return null; + } + return toPet((Map)rows.get(0)); + } + + public List listPets(String species) throws Exception { + final String wanted = species; + List rows = (List) withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + if(wanted == null || wanted.length() == 0) { + return db.query("SELECT id, name, species, weight, good FROM pet " + + "ORDER BY id", null); + } + return db.query("SELECT id, name, species, weight, good FROM pet " + + "WHERE species = ? ORDER BY id", new Object[]{wanted}); + } + }); + List out = new ArrayList(); + for(int iter = 0 ; iter < rows.size() ; iter++) { + out.add(toPet((Map)rows.get(iter))); + } + return out; + } + + public String setPhoto(long id, String data) throws Exception { + byte[] bytes = data == null ? new byte[0] : data.getBytes("UTF-8"); + final byte[] stored = bytes; + final long target = id; + int changed = ((Integer) withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + return new Integer(db.execute("UPDATE pet SET photo = ? WHERE id = ?", + new Object[]{stored, new Long(target)})); + } + })).intValue(); + if(changed == 0) { + throw new IllegalArgumentException("no pet " + id); + } + return "stored " + bytes.length + " bytes"; + } + + public String getPhoto(long id) throws Exception { + final long wanted = id; + List rows = (List) withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + return db.query("SELECT photo FROM pet WHERE id = ?", + new Object[]{new Long(wanted)}); + } + }); + if(rows.isEmpty()) { + return null; + } + Object photo = ((Map)rows.get(0)).get("photo"); + if(photo == null) { + return "no photo"; + } + // The point of the round trip: a BLOB column comes back as byte[], not as a + // lossy text rendering of the bytes. + byte[] bytes = (byte[])photo; + return "bytes=" + bytes.length + " content=" + new String(bytes, "UTF-8"); + } + + public String addPets(String authorization, final List pets) throws Exception { + requireCaller(authorization); + if(pets == null || pets.isEmpty()) { + throw new IllegalArgumentException("no pets given"); + } + Object inserted = inTransaction(new Db.Work() { + public Object run(Db conn) throws Exception { + int count = 0; + for(int iter = 0 ; iter < pets.size() ; iter++) { + Pet p = pets.get(iter); + if(p == null || p.name == null || p.name.length() == 0) { + // Throwing here rolls the whole batch back, including the + // rows already inserted in this loop. + throw new IllegalArgumentException("pet " + iter + " needs a name"); + } + conn.execute("INSERT INTO pet (name, species, weight, good) VALUES (?, ?, ?, ?)", + new Object[]{p.name, p.species, new Double(p.weight), + Boolean.valueOf(p.good)}); + count++; + } + return new Integer(count); + } + }); + return "inserted " + inserted; + } + + public String fetch(String url) throws Exception { + if(url == null || !url.startsWith("https://")) { + throw new IllegalArgumentException("only https URLs are fetched"); + } + Web.Result r = Web.get(url); + String body = r.getBodyAsString(); + if(body != null && body.length() > 120) { + body = body.substring(0, 120); + } + return "status=" + r.getStatus() + " body=" + body; + } + + public String deletePet(String authorization, long id) throws Exception { + requireCaller(authorization); + final long target = id; + int changed = ((Integer) withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + return new Integer(db.execute("DELETE FROM pet WHERE id = ?", + new Object[]{new Long(target)})); + } + })).intValue(); + return changed > 0 ? "deleted" : "not found"; + } + + private static Pet toPet(Map row) { + Pet p = new Pet(); + // Db hands every integer column back as Long and every real as Double, so + // the reads go through Number rather than casting to the field's type. + p.id = num(row.get("id")).longValue(); + p.name = str(row.get("name")); + p.species = str(row.get("species")); + p.weight = num(row.get("weight")).doubleValue(); + p.good = num(row.get("good")).longValue() != 0; + return p; + } + + private static Number num(Object v) { + return v instanceof Number ? (Number)v : new Long(0); + } + + private static String str(Object v) { + return v == null ? null : String.valueOf(v); + } +} diff --git a/vm/backend/demo/dbcheck/com/demo/DbCheck.java b/vm/backend/demo/dbcheck/com/demo/DbCheck.java new file mode 100644 index 00000000000..03b2114b28d --- /dev/null +++ b/vm/backend/demo/dbcheck/com/demo/DbCheck.java @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.codename1.backend.Database; + +/** + * Exercises the database layer against a REAL server, one engine per run. + * + * The same assertions run against SQLite, PostgreSQL and MySQL, which is the + * point: the Database facade claims a handler cannot tell which engine answered + * it, and the only way to hold that claim is to run one body of checks against + * all three and require the same answers. Value TYPES are asserted as well as + * values, because that is where the engines differ if nobody looks. + * + * Point it at a database with CN1_DBCHECK_URL. Without one it runs the SQLite + * arm only, which needs nothing installed. + */ +public class DbCheck { + private static int passed; + private static final List failures = new ArrayList(); + + public static void main(String[] args) throws Exception { + String url = System.getenv("CN1_DBCHECK_URL"); + if(url == null || url.length() == 0) { + url = ":memory:"; + note("CN1_DBCHECK_URL is unset, running the SQLite arm only"); + } + System.out.println("checking " + url); + Database db = Database.open(url); + try { + System.out.println("connected to " + db); + run(db, url); + } finally { + db.close(); + } + rejectsAnUntrustedCertificate(url); + + System.out.println("passed=" + passed + " failed=" + failures.size()); + for(int iter = 0 ; iter < failures.size() ; iter++) { + System.out.println("FAIL " + failures.get(iter)); + } + System.out.println(failures.isEmpty() ? "DBCHECK OK" : "DBCHECK FAILED"); + if(!failures.isEmpty()) { + System.exit(1); + } + } + + private static void run(Database db, String url) throws Exception { + boolean postgres = url.startsWith("postgres"); + boolean mysql = url.startsWith("mysql") || url.startsWith("mariadb"); + // Each engine spells "auto-incrementing primary key" and "binary blob" + // differently. Everything BELOW this line is identical for all three, + // which is the part being tested. + String key = postgres ? "id SERIAL PRIMARY KEY" + : (mysql ? "id BIGINT AUTO_INCREMENT PRIMARY KEY" + : "id INTEGER PRIMARY KEY AUTOINCREMENT"); + String blob = postgres ? "BYTEA" : (mysql ? "BLOB" : "BLOB"); + String real = postgres ? "DOUBLE PRECISION" : "DOUBLE"; + + db.execute("DROP TABLE IF EXISTS cn1_check", null); + db.execute("CREATE TABLE cn1_check (" + key + ", name VARCHAR(64), size " + real + + ", payload " + blob + ")", null); + + check("an insert reports one row", "1", String.valueOf(db.execute( + "INSERT INTO cn1_check (name, size, payload) VALUES (" + + placeholders(postgres, 3) + ")", + new Object[]{"first", Double.valueOf(1.5), bytes("hello")}))); + + db.execute("INSERT INTO cn1_check (name, size, payload) VALUES (" + + placeholders(postgres, 3) + ")", + new Object[]{"second", Double.valueOf(2.5), null}); + + List rows = db.query("SELECT id, name, size, payload FROM cn1_check ORDER BY id", + null); + check("both rows come back", "2", String.valueOf(rows.size())); + + Map first = (Map)rows.get(0); + // The TYPES are the contract, not just the values: a handler that gets a + // String where it got a Long on the other engine is broken by the switch. + check("an integer column is a Long", "java.lang.Long", typeOf(first.get("id"))); + check("a text column is a String", "java.lang.String", typeOf(first.get("name"))); + check("a real column is a Double", "java.lang.Double", typeOf(first.get("size"))); + check("a blob column is a byte[]", "byte[]", typeOf(first.get("payload"))); + check("the text value survives", "first", String.valueOf(first.get("name"))); + check("the real value survives", "1.5", String.valueOf(first.get("size"))); + check("the blob value survives", "hello", + new String((byte[])first.get("payload"), "UTF-8")); + + Map second = (Map)rows.get(1); + check("a NULL column is null", "null", String.valueOf(second.get("payload"))); + + // DECIMAL is the one type the three engines cannot be asked to agree + // on, because SQLite does not have it: a column DECLARED DECIMAL there + // has NUMERIC affinity and stores an INTEGER or a REAL, so there is no + // exact-decimal value to compare against. On the two servers that do + // have it, the column exists precisely because a double would not hold + // the value -- so it has to come back exact and it has to come back as + // a number the caller can read, not as a BLOB that JSON base64s. + if(postgres || mysql) { + String exact = "123456789012345678901234567890.12345"; + db.execute("DROP TABLE IF EXISTS cn1_check_decimal", null); + db.execute("CREATE TABLE cn1_check_decimal (amount DECIMAL(65,5))", null); + db.execute("INSERT INTO cn1_check_decimal (amount) VALUES (" + exact + ")", null); + List decimals = db.query("SELECT amount FROM cn1_check_decimal", null); + Object amount = ((Map)decimals.get(0)).get("amount"); + check("a decimal column is a String", "java.lang.String", typeOf(amount)); + check("the decimal value is exact", exact, String.valueOf(amount)); + } + + // Binding, not interpolation. A value containing a quote would end the + // statement early if this were concatenated. + db.execute("INSERT INTO cn1_check (name, size) VALUES (" + placeholders(postgres, 2) + ")", + new Object[]{"O'Brien; DROP TABLE cn1_check; --", Double.valueOf(0)}); + List quoted = db.query("SELECT name FROM cn1_check WHERE name = " + + placeholder(postgres, 1), new Object[]{"O'Brien; DROP TABLE cn1_check; --"}); + check("a quote in a bound value is data, not syntax", "1", + String.valueOf(quoted.size())); + + List counted = db.query("SELECT COUNT(*) AS total FROM cn1_check", null); + check("the table survived the injection attempt", "3", + String.valueOf(((Map)counted.get(0)).get("total"))); + + int updated = db.execute("UPDATE cn1_check SET size = " + placeholder(postgres, 1) + + " WHERE name = " + placeholder(postgres, 2), + new Object[]{Double.valueOf(9.5), "second"}); + check("an update reports the rows it changed", "1", String.valueOf(updated)); + + // A transaction that throws must leave nothing behind. + try { + db.transaction(new Database.Work() { + public Object run(Database inner) throws Exception { + inner.execute("INSERT INTO cn1_check (name, size) VALUES (" + + placeholders(inner.toString().startsWith("postgres"), 2) + ")", + new Object[]{"rolled-back", Double.valueOf(1)}); + throw new IllegalStateException("deliberate"); + } + }); + failures.add("a failing transaction must propagate its exception"); + } catch (IllegalStateException expected) { + passed++; + } + List afterRollback = db.query("SELECT id FROM cn1_check WHERE name = " + + placeholder(postgres, 1), new Object[]{"rolled-back"}); + check("a rolled-back insert left nothing", "0", String.valueOf(afterRollback.size())); + + Object committed = db.transaction(new Database.Work() { + public Object run(Database inner) throws Exception { + inner.execute("INSERT INTO cn1_check (name, size) VALUES (" + + placeholders(inner.toString().startsWith("postgres"), 2) + ")", + new Object[]{"committed", Double.valueOf(1)}); + return "done"; + } + }); + check("a transaction returns its body's value", "done", String.valueOf(committed)); + List afterCommit = db.query("SELECT id FROM cn1_check WHERE name = " + + placeholder(postgres, 1), new Object[]{"committed"}); + check("a committed insert is there", "1", String.valueOf(afterCommit.size())); + + // A statement error must be an exception, not a silent zero. + try { + db.query("SELECT no_such_column FROM cn1_check", null); + failures.add("a bad statement must throw"); + } catch (Exception expected) { + passed++; + } + // ...and the connection must still work afterwards, which is what a + // desynchronised protocol implementation gets wrong. + List afterError = db.query("SELECT COUNT(*) AS total FROM cn1_check", null); + check("the connection survives a statement error", "4", + String.valueOf(((Map)afterError.get(0)).get("total"))); + + if(!postgres) { + // PostgreSQL has no last-insert-id; the facade documents that it + // returns 0 there rather than pretending. + db.execute("INSERT INTO cn1_check (name, size) VALUES (" + + placeholders(postgres, 2) + ")", + new Object[]{"with-id", Double.valueOf(1)}); + check("the new row's id is reported", "true", + String.valueOf(db.lastInsertId() > 0)); + } else { + List returning = db.query("INSERT INTO cn1_check (name, size) VALUES ($1, $2) " + + "RETURNING id", new Object[]{"with-id", Double.valueOf(1)}); + check("RETURNING gives the new id", "true", + String.valueOf(((Map)returning.get(0)).get("id") != null)); + } + + db.execute("DROP TABLE cn1_check", null); + } + + /** + * The same URL with sslmode=require and no CA named must FAIL against a server + * whose certificate this host does not trust. + * + * Verification is the half of TLS that fails open: a client that encrypts and + * does not verify looks exactly like one that does, right up to the moment + * someone is in the middle. This check runs only when the URL under test named + * a CA, because that is precisely the case where the system store must not be + * enough. + */ + private static void rejectsAnUntrustedCertificate(String url) { + int at = url.indexOf("sslrootcert="); + if(at < 0) { + note("untrusted-certificate check skipped: this URL names no CA"); + return; + } + int end = url.indexOf('&', at); + String withoutCa = url.substring(0, at) + (end < 0 ? "" : url.substring(end + 1)); + try { + Database db = Database.open(withoutCa); + db.close(); + failures.add("a certificate signed by an untrusted CA was accepted"); + } catch (Exception expected) { + passed++; + } + } + + /** PostgreSQL numbers its placeholders; the other two use a question mark. */ + private static String placeholder(boolean postgres, int index) { + return postgres ? "$" + index : "?"; + } + + private static String placeholders(boolean postgres, int count) { + StringBuilder out = new StringBuilder(); + for(int iter = 1 ; iter <= count ; iter++) { + if(iter > 1) { + out.append(", "); + } + out.append(placeholder(postgres, iter)); + } + return out.toString(); + } + + private static String typeOf(Object value) { + if(value == null) { + return "null"; + } + if(value instanceof byte[]) { + return "byte[]"; + } + return value.getClass().getName(); + } + + private static byte[] bytes(String value) throws Exception { + return value.getBytes("UTF-8"); + } + + private static void check(String name, String expected, String actual) { + if(expected.equals(actual)) { + passed++; + } else { + failures.add(name + ": expected <" + expected + "> but was <" + actual + ">"); + } + } + + private static void note(String message) { + System.out.println("NOTE " + message); + } +} diff --git a/vm/backend/demo/gcpause/com/demo/GcPause.java b/vm/backend/demo/gcpause/com/demo/GcPause.java new file mode 100644 index 00000000000..151dccf210c --- /dev/null +++ b/vm/backend/demo/gcpause/com/demo/GcPause.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +/** + * The narrowest test of the thing the server benchmark kept pointing at: how + * long does a mutator stop when the collector runs? + * + * No sockets, no HTTP, no scheduler -- one thread allocating short-lived objects + * against a fixed live set, timing EVERY iteration. Steady work per iteration + * means every large gap is the collector and nothing else, so the distribution's + * tail IS the pause distribution. The Go twin (gcpause.go) is the same loop with + * the same live-set size and iteration count. + * + * Reported as a log2 histogram rather than a mean: a mean over millions of fast + * iterations hides exactly the rare multi-millisecond stall this exists to find. + */ +public class GcPause { + static final class Node { + int v; + Node next; + Node(int v, Node next) { this.v = v; this.next = next; } + } + + static int envInt(String name, int def) { + String v = System.getenv(name); + if(v == null || v.length() == 0) { + return def; + } + try { + return Integer.parseInt(v); + } catch (NumberFormatException err) { + return def; + } + } + + public static void main(String[] args) { + int iterations = envInt("ITERS", 20000000); + int liveSize = envInt("LIVE", 4096); // power of two, for the mask + Node[] live = new Node[liveSize]; + long[] buckets = new long[48]; + long worst = 0; + long checksum = 0; + + // Untimed warm-up so first-touch page faults and the first collection are + // not charged to the measurement. + for(int i = 0 ; i < 1000000 ; i++) { + live[i & (liveSize - 1)] = new Node(i, null); + } + + long prev = System.nanoTime(); + for(int i = 0 ; i < iterations ; i++) { + // Each new node points at an older live one, so the collector has a + // real graph to trace rather than a field of isolated leaves. + Node n = new Node(i, live[(i * 7) & (liveSize - 1)]); + live[i & (liveSize - 1)] = n; + checksum += n.v; + long now = System.nanoTime(); + long d = now - prev; + prev = now; + int b = 0; + long x = d; + while(x > 0 && b < 47) { + x >>= 1; + b++; + } + buckets[b]++; + if(d > worst) { + worst = d; + } + } + report(buckets, worst, checksum, iterations); + } + + static void report(long[] buckets, long worst, long checksum, long iterations) { + System.out.println("GCPAUSE iterations=" + iterations + " checksum=" + checksum); + System.out.println("GCPAUSE maxNs=" + worst); + long total = 0; + for(int b = 0 ; b < buckets.length ; b++) { + total += buckets[b]; + } + printPercentile("p50", buckets, total, 0.50); + printPercentile("p99", buckets, total, 0.99); + printPercentile("p999", buckets, total, 0.999); + printPercentile("p9999", buckets, total, 0.9999); + // Everything at or above 64us: with steady per-iteration work nothing but + // a collection reaches that, so this counts pauses directly. + long stalls = 0; + for(int b = 17 ; b < buckets.length ; b++) { + stalls += buckets[b]; + } + System.out.println("GCPAUSE stallsOver64us=" + stalls); + for(int b = 17 ; b < buckets.length ; b++) { + if(buckets[b] != 0) { + System.out.println("GCPAUSE bucket=" + (1L << (b - 1)) + "ns count=" + buckets[b]); + } + } + } + + static void printPercentile(String name, long[] buckets, long total, double q) { + long want = (long)(q * (double)total); + long seen = 0; + for(int b = 0 ; b < buckets.length ; b++) { + seen += buckets[b]; + if(seen > want) { + System.out.println("GCPAUSE " + name + "Ns=" + (b == 0 ? 0L : (1L << (b - 1)))); + return; + } + } + } +} diff --git a/vm/backend/demo/gcstress/com/demo/GcStress.java b/vm/backend/demo/gcstress/com/demo/GcStress.java new file mode 100644 index 00000000000..33c7b3ecd54 --- /dev/null +++ b/vm/backend/demo/gcstress/com/demo/GcStress.java @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +/** + * A deep stress case for the PARALLEL mark path. + * + * gcMarkResolveThreadCount forces one marker because arm64 Linux was corrupting + * the heap with the pool enabled and a second ordering hole was never found. + * demo/gcpause does not reach it: one mutator, one reference per object, and no + * mutation while the mark runs. Everything here exists to attack a concurrent + * marker specifically. + * + * - Several mutator threads, so marking overlaps real mutation. + * - REWIRING of reference fields while the collector is tracing, which is what + * the SATB barrier exists for: a reference moved from an unscanned object to + * a scanned one is exactly the object a snapshot collector loses. + * - Resurrection through a shared stash: objects go unreachable and reachable + * again across threads, so a marker that claims an object without publishing + * its children shows up as a freed-but-live object. + * - Mixed shapes (object, array, String) so more than one markFunction runs, + * and DEEP chains so the mark worklist overflows into the grace pass. + * + * Detection does not rely on a crash. Every node carries a magic word and a + * payload whose checksum is derived from its identity, so a prematurely freed + * and reused node is caught by a value check even when it does not segfault. + */ +public class GcStress { + static final int MAGIC = 0x5A5AC0DE; + + static final class Node { + int magic; + int id; + int[] payload; + String name; + Node left; + Node right; + + Node(int id) { + this.magic = MAGIC; + this.id = id; + this.payload = new int[8]; + for(int i = 0 ; i < payload.length ; i++) { + payload[i] = id + i; + } + this.name = "node-" + id; + } + + /** Non-zero describes the damage, so a failure says what was wrong. */ + String check() { + if(magic != MAGIC) { + return "magic=" + Integer.toHexString(magic) + " id=" + id; + } + if(payload == null || payload.length != 8) { + return "payload shape id=" + id; + } + for(int i = 0 ; i < 8 ; i++) { + if(payload[i] != id + i) { + return "payload[" + i + "]=" + payload[i] + " want " + (id + i); + } + } + if(name == null || !name.equals("node-" + id)) { + return "name=" + name + " id=" + id; + } + return null; + } + } + + /** Cross-thread visibility of the graph is the point, hence the shared stash. */ + static final Object STASH_LOCK = new Object(); + static Node[] stash = new Node[512]; + static volatile boolean running = true; + static volatile String failure = null; + static int nextId = 1; + + static synchronized int allocId() { + return nextId++; + } + + static int envInt(String name, int def) { + String v = System.getenv(name); + if(v == null || v.length() == 0) { + return def; + } + try { + return Integer.parseInt(v); + } catch (NumberFormatException err) { + return def; + } + } + + static final class Worker extends Thread { + private final int seed; + private final int rounds; + Worker(int seed, int rounds) { + this.seed = seed; + this.rounds = rounds; + } + + public void run() { + int rnd = seed * 0x9E3779B1 + 1; // hex form: decimal would overflow int + Node[] local = new Node[256]; + try { + for(int round = 0 ; round < rounds && running ; round++) { + rnd = rnd * 1103515245 + 12345; + int slot = (rnd >>> 8) & 255; + + // A short chain per round: depth makes the mark recurse and the + // worklist overflow rather than fitting in one batch. + Node head = new Node(allocId()); + Node cur = head; + for(int d = 0 ; d < 12 ; d++) { + cur.left = new Node(allocId()); + cur.right = new Node(allocId()); + cur = cur.left; + } + local[slot] = head; + + // Rewire an older node's child to a newer one WHILE the collector + // may be tracing: the deletion barrier has to catch the old value. + int other = (rnd >>> 16) & 255; + Node victim = local[other]; + if(victim != null) { + victim.right = head; + } + + // Publish and take back through shared state, so objects change + // reachability across threads mid-cycle. + if((round & 7) == 0) { + synchronized(STASH_LOCK) { + int si = (rnd >>> 4) & 511; + Node taken = stash[si]; + stash[si] = head; + if(taken != null) { + local[(other + 1) & 255] = taken; + } + } + } + + // Drop references so most of it is garbage. + if((round & 3) == 0) { + local[(slot + 7) & 255] = null; + } + + // Verify what we still hold. A prematurely collected node shows + // up here as damaged content rather than as a crash. + if((round & 15) == 0) { + for(int i = 0 ; i < local.length ; i++) { + Node n = local[i]; + int depth = 0; + while(n != null && depth < 6) { + String bad = n.check(); + if(bad != null) { + failure = "worker" + seed + " " + bad; + running = false; + return; + } + n = n.left; + depth++; + } + } + } + } + } catch (Throwable err) { + failure = "worker" + seed + " threw " + err; + running = false; + } + } + } + + public static void main(String[] args) throws Exception { + int threads = envInt("THREADS", 4); + int rounds = envInt("ROUNDS", 4000); + int gcEvery = envInt("GC_EVERY_MS", 40); + + Worker[] workers = new Worker[threads]; + for(int i = 0 ; i < threads ; i++) { + workers[i] = new Worker(i + 1, rounds); + workers[i].start(); + } + + // Keep collections frequent so mark overlaps mutation for most of the run. + int cycles = 0; + while(running) { + boolean alive = false; + for(int i = 0 ; i < threads ; i++) { + if(workers[i].isAlive()) { + alive = true; + break; + } + } + if(!alive) { + break; + } + System.gc(); + cycles++; + Thread.sleep(gcEvery); + } + for(int i = 0 ; i < threads ; i++) { + workers[i].join(); + } + + // Final sweep over everything still reachable from the stash. + String bad = null; + synchronized(STASH_LOCK) { + for(int i = 0 ; i < stash.length && bad == null ; i++) { + Node n = stash[i]; + int depth = 0; + while(n != null && depth < 12 && bad == null) { + bad = n.check(); + n = n.left; + depth++; + } + } + } + if(bad != null && failure == null) { + failure = "final " + bad; + } + if(failure != null) { + System.out.println("GCSTRESS FAIL " + failure); + System.exit(1); + } + System.out.println("GCSTRESS OK threads=" + threads + " rounds=" + rounds + + " gcCycles=" + cycles + " ids=" + nextId); + } +} diff --git a/vm/backend/demo/mapbench/com/demo/MapBench.java b/vm/backend/demo/mapbench/com/demo/MapBench.java new file mode 100644 index 00000000000..767d8bd2af3 --- /dev/null +++ b/vm/backend/demo/mapbench/com/demo/MapBench.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * HashMap against LinkedHashMap on the translated target. + * + * HashMap's get/put/remove are NATIVE in ParparVM (open addressing over parallel + * arrays, see nativeMethods.m). LinkedHashMap extends it but overrides exactly + * those methods in Java to maintain its ordering links, so it cannot reach the C + * fast path -- and its put allocates a CompactEntry per call to hand to + * removeEldestEntry. This measures what that costs. + */ +public class MapBench { + static final int N = 4; // the shape a JSON codec builds + static final int ITERS = 2000000; + + static long fill(boolean linked) { + long t0 = System.nanoTime(); + for (int i = 0; i < ITERS; i++) { + Map m = linked ? new LinkedHashMap() : new HashMap(); + m.put("name", "value"); + m.put("email", "x@example.com"); + m.put("id", new Long(42)); + m.put("active", Boolean.TRUE); + if (m.size() != N) { + throw new IllegalStateException("bad size"); + } + } + return System.nanoTime() - t0; + } + + static long lookup(boolean linked) { + Map m = linked ? new LinkedHashMap() : new HashMap(); + m.put("name", "value"); + m.put("email", "x@example.com"); + m.put("id", new Long(42)); + m.put("active", Boolean.TRUE); + long t0 = System.nanoTime(); + long sink = 0; + for (int i = 0; i < ITERS; i++) { + if (m.get("email") != null) { + sink++; + } + } + if (sink != ITERS) { + throw new IllegalStateException("bad sink"); + } + return System.nanoTime() - t0; + } + + public static void main(String[] args) { + fill(false); fill(true); lookup(false); lookup(true); // warm + for (int rep = 1; rep <= 3; rep++) { + long hf = fill(false), lf = fill(true); + long hg = lookup(false), lg = lookup(true); + System.out.println("rep" + rep + + " build: hash=" + (hf / ITERS) + "ns linked=" + (lf / ITERS) + + "ns (" + (lf * 100 / hf) + "% of hash)" + + " get: hash=" + (hg / ITERS) + "ns linked=" + (lg / ITERS) + + "ns (" + (lg * 100 / hg) + "%)"); + } + } +} diff --git a/vm/backend/demo/petserver/com/demo/PetServer.java b/vm/backend/demo/petserver/com/demo/PetServer.java new file mode 100644 index 00000000000..003eec81eb9 --- /dev/null +++ b/vm/backend/demo/petserver/com/demo/PetServer.java @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.LinkedHashMap; +import java.util.Map; + +import com.codename1.backend.Db; +import com.codename1.backend.DbPool; +import com.codename1.backend.HttpServer; +import com.codename1.backend.Json; +import com.codename1.backend.Signals; +import com.codename1.backend.StaticFiles; +import com.codename1.backend.Tls; + +/** + * The same contract, the same service, a different front end. + * + * Nothing about GreeterApi, GreeterApiDispatcher, PetJson or GreeterService knows + * whether it is behind a Lambda or a socket -- which is the point of generating the + * dispatcher from the contract rather than writing a router per deployment. Greeter + * (the Lambda) and this file are the only two things that differ, and both are + * transport glue. + */ +public class PetServer { + public static void main(String[] args) throws Exception { + Signals.installShutdownHandler(); + int port = envInt("CN1_PORT", 8080); + int workers = envInt("CN1_WORKERS", 16); + String dbPath = System.getenv("CN1_DB_PATH"); + + final DbPool pool; + final GreeterService service; + if(dbPath == null || ":memory:".equals(dbPath)) { + // An in-memory database cannot be pooled: each connection would get its + // own. One shared connection is correct here, and SQLite serializes it. + pool = null; + service = new GreeterService(Db.open(":memory:")); + } else { + // The POOL, not one connection out of it. Borrowing one here and sharing + // it left the rest of the pool idle and let concurrent requests interleave + // on the same connection -- a plain insert could land inside another + // request's transaction and be rolled back with it. + pool = DbPool.open(dbPath, Math.max(2, workers / 4), 5000); + service = new GreeterService(pool); + } + final GreeterApiDispatcher dispatcher = new GreeterApiDispatcher(service); + + // Static files are served from CN1_STATIC_ROOT when it is set. They are + // tried only AFTER the API, so a file can never shadow a route. + String staticRoot = System.getenv("CN1_STATIC_ROOT"); + final StaticFiles files = staticRoot == null ? null + : new StaticFiles(staticRoot, "/static", "index.html", "public, max-age=3600"); + if(files != null) { + System.out.println("serving " + staticRoot + " at /static" + + (StaticFiles.isZeroCopy() ? " (sendfile)" : " (read/write)")); + } + + // TLS is terminated here when a certificate is configured. Plaintext is the + // right default behind a load balancer that already terminated it. + String certPath = System.getenv("CN1_TLS_CERT"); + String keyPath = System.getenv("CN1_TLS_KEY"); + // HTTP/2 is advertised through ALPN; there is no other way to reach it over + // TLS. Set CN1_HTTP2=0 to offer only http/1.1. + boolean offerHttp2 = !"0".equals(System.getenv("CN1_HTTP2")); + Tls tls = certPath == null || keyPath == null ? null + : Tls.create(certPath, keyPath, offerHttp2); + + // The handler needs the server to report its own metrics, and the server + // needs the handler to be constructed: one holder breaks the cycle. + final HttpServer[] serverRef = new HttpServer[1]; + final HttpServer server = HttpServer.start(null, port, 512, workers, new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + String method = request.getMethod(); + String target = request.getTarget(); + if("/healthz".equals(stripQuery(target))) { + return HttpServer.Response.json(200, Json.write(serverRef[0].getMetrics())); + } + // The DEFERRED json form: respondJson hands the value over + // unserialised so the HTTP/1 writer can render it straight into + // the connection buffer, which leaves response.body empty. Any + // code that measures that array instead of rendering the value + // reports zero for a representation that is not, and only a + // handler shaped like this one can show it. + if("/deferred".equals(stripQuery(target))) { + Map value = new LinkedHashMap(); + value.put("name", "deferred"); + value.put("digits", "1234567890"); + return request.respondJson(200, value); + } + // Deliberately a body on a status that cannot carry one. A handler + // is allowed to build this -- the Response constructor takes any + // status and any bytes -- and suppressing it is the server's job, + // because writing it would leave the client reading those bytes as + // the start of the next reply on a keep-alive connection. + if("/nocontent".equals(stripQuery(target))) { + return new HttpServer.Response(204, "text/plain", + "junk".getBytes("UTF-8")); + } + // Also deliberately malformed, and for the same reason: a handler + // can put anything in extraHeaders, and what it must never do is + // reach the wire. A name with a space in it is not a field name, + // and a name with a LEADING space is obsolete line folding, which + // appends both to whatever header came before -- so a header the + // handler could not have meant would silently rewrite one the + // server owns. + // 205 with bytes, for the same reason /nocontent exists: a + // handler may build it, and RFC 9110 ends a Reset Content + // response at the header section, so writing them would leave a + // keep-alive client reading them as the next reply. + // Echoes the VERB back, so a client can prove which one arrived + // rather than which one it believes it sent. HttpServer routes + // seven methods; this answers for any of them. + if("/echo".equals(stripQuery(target))) { + String echoed = request.getBody(); + return new HttpServer.Response(200, "text/plain", + ("method=" + method + " len=" + + (echoed == null ? 0 : echoed.length())).getBytes("UTF-8")); + } + // A body whose size the caller picks, so a test can ask for more + // than one serialisation buffer's worth and check that every byte + // still arrives. Deliberately NOT file-backed: the point is the + // in-memory DATA-frame path, which is where the output buffer sits. + if("/bulk".equals(stripQuery(target))) { + String sizeText = request.queryParam("size"); + int size = sizeText == null ? 1024 : Integer.parseInt(sizeText); + byte[] payload = new byte[size]; + for(int iter = 0 ; iter < size ; iter++) { + payload[iter] = (byte)('a' + (iter % 26)); + } + return new HttpServer.Response(200, "text/plain", payload); + } + if("/reset".equals(stripQuery(target))) { + return new HttpServer.Response(205, "text/plain", + "junk".getBytes("UTF-8")); + } + if("/rawheader".equals(stripQuery(target))) { + Map extra = new LinkedHashMap(); + extra.put("X-Good", "ok"); + extra.put("X Bad", "space-in-name"); + extra.put(" X-Fold", "obsolete-folding"); + extra.put("X:Colon", "colon-in-name"); + return new HttpServer.Response(200, "text/plain", + "raw".getBytes("UTF-8"), extra); + } + // A non-ASCII parameter NAME, spelled as an escape so this source + // stays ASCII. Every client percent-encodes such a name as its + // UTF-8 octets, so the server has to compare it that way round. + if("/accent".equals(stripQuery(target))) { + String value = request.queryParam("caf\u00e9"); + return new HttpServer.Response(200, "text/plain", + ("caf\u00e9=" + value).getBytes("UTF-8")); + } + if(!dispatcher.hasRoute(method, target)) { + if(files != null) { + HttpServer.Response served = files.handle(request); + if(served != null) { + return served; + } + } + return HttpServer.Response.json(404, + "{\"error\":\"no route for " + method + " " + target + "\"}"); + } + Object body = decodeBody(request.getBody()); + Object result; + try { + result = dispatcher.dispatch(method, target, request.getHeaders(), body); + } catch (SecurityException err) { + // Authentication or authorisation failed. A 500 here would be + // both wrong and unactionable for the client. + return HttpServer.Response.json(401, errorJson(err.getMessage())); + } catch (IllegalArgumentException err) { + // The handler rejected the input; that is a 400, not a 500. + return HttpServer.Response.json(400, errorJson(err.getMessage())); + } + if(result == null) { + return HttpServer.Response.json(404, "{\"error\":\"not found\"}"); + } + return HttpServer.Response.json(200, Json.write(result)); + } + }, tls); + serverRef[0] = server; + System.out.println("listening on port " + server.getPort() + + " with " + workers + " workers" + + (tls == null ? " (plaintext)" : " (TLS)")); + Signals.onShutdown(new Runnable() { + public void run() { + server.stop(10000); + if(pool != null) { + pool.close(); + } + System.out.println("stopped"); + // Signals ends the process once this returns. Exiting from here + // would deadlock under the JavaSE implementation, where the same + // body runs from a JVM shutdown hook. + } + }); + // Hold main here. The reactor and workers are detached threads, so a main + // that returns ends the process with status 0 and no message. + server.awaitTermination(); + } + + private static String stripQuery(String target) { + int q = target == null ? -1 : target.indexOf('?'); + return q < 0 ? target : target.substring(0, q); + } + + private static Object decodeBody(String raw) { + if(raw == null || raw.length() == 0) { + return null; + } + try { + return Json.parse(raw); + } catch (Exception err) { + // Not JSON: hand it through as text so a @Body String still works. + return raw; + } + } + + private static String errorJson(String message) { + Map out = new LinkedHashMap(); + out.put("error", message == null ? "bad request" : message); + return Json.write(out); + } + + private static int envInt(String name, int fallback) { + String v = System.getenv(name); + if(v == null || v.length() == 0) { + return fallback; + } + try { + return Integer.parseInt(v.trim()); + } catch (NumberFormatException err) { + return fallback; + } + } +} diff --git a/vm/backend/demo/petstore/com/demo/Greeter.java b/vm/backend/demo/petstore/com/demo/Greeter.java new file mode 100644 index 00000000000..e04d739cc7d --- /dev/null +++ b/vm/backend/demo/petstore/com/demo/Greeter.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.LinkedHashMap; +import java.util.Map; + +import com.codename1.backend.Db; +import com.codename1.backend.Handler; +import com.codename1.backend.Json; +import com.codename1.backend.LambdaRuntime; + +/** + * The MVP function. Note what is NOT here: no route table, no path parsing, no + * argument extraction, no DTO marshalling. GreeterApiDispatcher and PetJson are + * generated from the shared GreeterApi contract, so adding a parameter to the + * contract breaks this build instead of returning the wrong thing at runtime. + * + * What is left is the transport glue: decode the host's event envelope, hand the + * dispatcher a decoded body, encode whatever comes back. + */ +public class Greeter { + public static void main(String[] args) { + final GreeterApiDispatcher dispatcher; + try { + // CN1_DB_PATH lets a test point at a scratch file; a real function would + // use the writable path its host gives it (/tmp on Lambda), and ":memory:" + // is the honest default for a stateless invocation. + String dbPath = System.getenv("CN1_DB_PATH"); + Db db = Db.open(dbPath == null ? ":memory:" : dbPath); + dispatcher = new GreeterApiDispatcher(new GreeterService(db)); + } catch (Exception err) { + System.err.println("Could not start: " + err); + return; + } + LambdaRuntime.run(new Handler() { + public String handle(String event, String requestId) throws Exception { + Map envelope; + try { + envelope = Json.parseObject(event); + } catch (Exception err) { + return error(400, "malformed event: " + err.getMessage()); + } + String method = string(envelope.get("httpMethod")); + String path = string(envelope.get("path")); + if(method == null || path == null) { + return error(400, "expected httpMethod and path"); + } + Map headers = envelope.get("headers") instanceof Map + ? (Map)envelope.get("headers") : null; + Object body = decodeBody(envelope.get("body")); + + if(!dispatcher.hasRoute(method, path)) { + return error(404, "no route for " + method + " " + path); + } + Object result; + try { + result = dispatcher.dispatch(method, path, headers, body); + } catch (SecurityException err) { + // Authentication or authorisation failed; not a server fault. + return error(401, err.getMessage()); + } catch (IllegalArgumentException err) { + // The handler rejected the input; that is a 400, not a 500. + return error(400, err.getMessage()); + } catch (Exception err) { + System.err.println("[" + requestId + "] " + err); + return error(500, err.getClass().getName()); + } + Map out = new LinkedHashMap(); + out.put("statusCode", new Integer(result == null ? 404 : 200)); + out.put("body", result == null ? "not found" : Json.write(result)); + return Json.write(out); + } + }); + } + + /** + * The envelope carries the body as a JSON string, so it is decoded here rather + * than in the dispatcher - the dispatcher deals in values, not transport. + */ + private static Object decodeBody(Object raw) { + if(raw == null) { + return null; + } + String text = String.valueOf(raw); + if(text.length() == 0) { + return null; + } + try { + return Json.parse(text); + } catch (Exception err) { + // Not JSON: hand it through as text so a @Body String still works. + return text; + } + } + + private static String string(Object v) { + return v == null ? null : String.valueOf(v); + } + + private static String error(int status, String message) { + Map out = new LinkedHashMap(); + out.put("statusCode", new Integer(status)); + out.put("body", message); + return Json.write(out); + } +} diff --git a/vm/backend/demo/poolcheck/com/demo/PoolCheck.java b/vm/backend/demo/poolcheck/com/demo/PoolCheck.java new file mode 100644 index 00000000000..10f3aa22f74 --- /dev/null +++ b/vm/backend/demo/poolcheck/com/demo/PoolCheck.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.List; +import java.util.Map; + +import com.codename1.backend.Db; +import com.codename1.backend.DbPool; + +/** + * Exercises DbPool from several threads at once against one WAL database, which + * is the arrangement a pool exists for. Verifies the row count rather than just + * that nothing threw: a pool that silently gave two threads the same connection + * would still "work" until it corrupted a result. + */ +public class PoolCheck { + private static final int THREADS = 8; + private static final int PER_THREAD = 50; + + public static void main(String[] args) throws Exception { + String path = System.getenv("CN1_DB_PATH"); + if(path == null) { + System.out.println("SKIP: set CN1_DB_PATH"); + return; + } + final DbPool pool = DbPool.open(path, 4, 5000); + Db setup = pool.borrow(); + setup.execute("DROP TABLE IF EXISTS counter", null); + setup.execute("CREATE TABLE counter (id INTEGER PRIMARY KEY AUTOINCREMENT, who TEXT, n INTEGER)", null); + pool.release(setup); + + final int[] failures = new int[1]; + Thread[] workers = new Thread[THREADS]; + for(int t = 0 ; t < THREADS ; t++) { + final String who = "worker-" + t; + workers[t] = new Thread(new Runnable() { + public void run() { + for(int i = 0 ; i < PER_THREAD ; i++) { + final int n = i; + try { + pool.inTransaction(new Db.Work() { + public Object run(Db db) throws Exception { + db.execute("INSERT INTO counter (who, n) VALUES (?, ?)", + new Object[]{who, new Integer(n)}); + return null; + } + }); + } catch (Exception err) { + synchronized(failures) { + failures[0]++; + } + System.err.println(who + " failed: " + err); + } + } + } + }); + workers[t].start(); + } + for(int t = 0 ; t < THREADS ; t++) { + workers[t].join(); + } + + Db check = pool.borrow(); + List rows = check.query("SELECT COUNT(*) AS c FROM counter", null); + long count = ((Number)((Map)rows.get(0)).get("c")).longValue(); + List distinct = check.query("SELECT COUNT(DISTINCT who) AS c FROM counter", null); + long writers = ((Number)((Map)distinct.get(0)).get("c")).longValue(); + pool.release(check); + pool.close(); + + int expected = THREADS * PER_THREAD; + System.out.println("rows=" + count + " expected=" + expected + + " writers=" + writers + " failures=" + failures[0]); + System.out.println(count == expected && writers == THREADS && failures[0] == 0 + ? "POOL OK" : "POOL FAILED"); + } +} diff --git a/vm/backend/demo/reactorcheck/com/demo/ReactorCheck.java b/vm/backend/demo/reactorcheck/com/demo/ReactorCheck.java new file mode 100644 index 00000000000..e8a01350401 --- /dev/null +++ b/vm/backend/demo/reactorcheck/com/demo/ReactorCheck.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import com.codename1.backend.Reactor; +import com.codename1.backend.ServerSocket; +import com.codename1.backend.Tcp; + +/** + * Smoke test for the poller: bind, register, accept, register the accepted + * connection, read from it. Small on purpose -- when the HTTP server went silent + * this is what separated "the reactor never reports readiness" from "the server + * has a bug", and the answer was the latter. + * + * Registration happens on the main thread and polling on another, because that is + * how HttpServer uses it. + */ +public class ReactorCheck { + public static void main(String[] args) throws Exception { + final ServerSocket listener = ServerSocket.bind(null, 0, 16); + final int port = listener.getPort(); + final Reactor reactor = Reactor.create(); + ServerSocket.setBlocking(listener.getFd(), false); + reactor.add(listener.getFd(), Reactor.READ); + System.out.println("listening on " + port + " fd=" + listener.getFd()); + + new Thread(new Runnable() { + public void run() { + try { + Thread.sleep(300); + Tcp t = Tcp.connect("127.0.0.1", port, 0); + byte[] hello = "GET /x HTTP/1.1\r\n\r\n".getBytes("UTF-8"); + t.write(hello, 0, hello.length); + Thread.sleep(2000); + t.close(); + } catch (Exception err) { + System.out.println("client failed: " + err); + } + } + }).start(); + + int[] ready = new int[16]; + for (int round = 0; round < 8; round++) { + int n = reactor.await(ready, 1000); + if (n <= 0) { + continue; + } + if (ready[0] == listener.getFd()) { + int client = listener.accept(); + if (client >= 0) { + ServerSocket.setBlocking(client, false); + reactor.add(client, Reactor.READ); + System.out.println("accepted fd=" + client); + } + continue; + } + byte[] buf = new byte[256]; + ServerSocket.setBlocking(ready[0], true); + int got = ServerSocket.read(ready[0], buf, 0, buf.length); + System.out.println("read " + got + " bytes: " + + (got > 0 ? new String(buf, 0, got, "UTF-8").trim() : "")); + System.out.println(got > 0 ? "REACTOR OK" : "REACTOR FAILED: empty read"); + return; + } + System.out.println("REACTOR FAILED: no readiness reported"); + } +} diff --git a/vm/backend/demo/s3check/com/demo/S3Check.java b/vm/backend/demo/s3check/com/demo/S3Check.java new file mode 100644 index 00000000000..46bf2795f9d --- /dev/null +++ b/vm/backend/demo/s3check/com/demo/S3Check.java @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.ArrayList; +import java.util.List; + +import com.codename1.backend.Web; +import com.codename1.backend.aws.Aws; +import com.codename1.backend.aws.Credentials; +import com.codename1.backend.aws.S3; + +/** + * Exercises SigV4 and the S3 client against a REAL S3-compatible server. + * + * Signature code cannot be tested against itself. A wrong canonical form -- a + * misencoded space, an unsorted query parameter, a header case -- produces a + * signature this code agrees with completely and the service rejects with a bare + * 403. So the checks below run against a server (MinIO locally, and any + * S3-compatible endpoint in CI) and the KNOWN-ANSWER vectors from the AWS + * documentation run everywhere, because those pin the canonical form itself. + * + * Point it at a server with CN1_S3CHECK_ENDPOINT / _KEY / _SECRET / _BUCKET. + * Without one only the known-answer vectors run. + */ +public class S3Check { + private static int passed; + private static final List failures = new ArrayList(); + + public static void main(String[] args) throws Exception { + knownAnswers(); + liveServer(); + + System.out.println("passed=" + passed + " failed=" + failures.size()); + for(int iter = 0 ; iter < failures.size() ; iter++) { + System.out.println("FAIL " + failures.get(iter)); + } + System.out.println(failures.isEmpty() ? "S3CHECK OK" : "S3CHECK FAILED"); + if(!failures.isEmpty()) { + System.exit(1); + } + } + + /** + * The vectors AWS publishes for SigV4, which fix the canonical form + * independently of any server. These are the checks that say WHICH part is + * wrong when a live request comes back 403. + */ + private static void knownAnswers() throws Exception { + // The documented derivation for the key AWS uses in its own examples. + byte[] key = Aws.signingKey("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + "20150830", "us-east-1", "iam"); + check("the signing key matches the published vector", + "c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9", + Aws.hex(key)); + + // Percent-encoding: a space is %20 and never '+', '~' is left alone, and + // the hex is upper case. All three differ from URLEncoder, and each one on + // its own is a 403. + check("a space encodes as %20", "a%20b", Aws.encode("a b")); + check("a tilde is not encoded", "~", Aws.encode("~")); + check("a slash inside a segment is encoded", "a%2Fb", Aws.encode("a/b")); + check("a slash between segments is not", "/a/b%20c", Aws.encodePath("/a/b c")); + check("non-ASCII is UTF-8 percent encoded", "%C3%A9", Aws.encode("\u00e9")); + + // Query parameters are sorted by their ENCODED name. + java.util.Map query = new java.util.LinkedHashMap(); + query.put("marker", "b"); + query.put("acl", ""); + query.put("Prefix", "a b"); + check("query parameters are sorted and encoded", + "Prefix=a%20b&acl=&marker=b", Aws.canonicalQuery(query)); + + check("an empty body hashes to the documented value", + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + Aws.sha256Hex(new byte[0])); + + // Header values have their internal whitespace collapsed. + check("header whitespace is collapsed", "a b", Aws.collapse(" a b ")); + } + + private static void liveServer() throws Exception { + String endpoint = System.getenv("CN1_S3CHECK_ENDPOINT"); + if(endpoint == null || endpoint.length() == 0) { + System.out.println("NOTE live S3 checks skipped: set CN1_S3CHECK_ENDPOINT"); + return; + } + String bucket = System.getenv("CN1_S3CHECK_BUCKET"); + if(bucket == null || bucket.length() == 0) { + bucket = "cn1-backend-check"; + } + Credentials credentials = new Credentials(System.getenv("CN1_S3CHECK_KEY"), + System.getenv("CN1_S3CHECK_SECRET"), null); + S3 s3 = S3.forEndpoint(credentials, System.getenv("CN1_S3CHECK_REGION"), endpoint); + System.out.println("checking s3://" + bucket + " at " + endpoint); + s3.createBucket(bucket); + + String key = "folder/an object with spaces & symbols.txt"; + byte[] content = "hello from the backend".getBytes("UTF-8"); + + String etag = s3.putObject(bucket, key, content, "text/plain"); + check("a put returns an etag", "true", String.valueOf(etag.length() > 0)); + + byte[] fetched = s3.getObject(bucket, key); + check("the object round trips", new String(content, "UTF-8"), + new String(fetched, "UTF-8")); + + S3.ObjectInfo info = s3.headObject(bucket, key); + check("head reports the size", String.valueOf(content.length), + String.valueOf(info.getSize())); + check("head reports the content type", "text/plain", info.getContentType()); + + check("head on a missing key is null", "null", + String.valueOf(s3.headObject(bucket, "no/such/key"))); + + List listed = s3.listObjects(bucket, "folder/", 100); + check("the key is listed", "true", String.valueOf(listed.contains(key))); + + // A presigned URL is the whole point of this for a mobile client: it must + // work with NO credentials on the request. + String url = s3.presignGet(bucket, key, 300); + Web.Result direct = Web.request("GET", url, null, null); + check("a presigned GET works unauthenticated", "200", + String.valueOf(direct.getStatus())); + check("a presigned GET returns the object", new String(content, "UTF-8"), + direct.getBodyAsString()); + + // ...and must stop working when tampered with, or it is not a signature. + Web.Result tampered = Web.request("GET", url.substring(0, url.length() - 1) + "0", + null, null); + check("a tampered presigned URL is rejected", "true", + String.valueOf(tampered.getStatus() >= 400)); + + s3.deleteObject(bucket, key); + check("the object is gone after delete", "null", + String.valueOf(s3.headObject(bucket, key))); + } + + private static void check(String name, String expected, String actual) { + if(expected.equals(actual)) { + passed++; + } else { + failures.add(name + ": expected <" + expected + "> but was <" + actual + ">"); + } + } +} diff --git a/vm/backend/demo/selftest/com/demo/SelfTest.java b/vm/backend/demo/selftest/com/demo/SelfTest.java new file mode 100644 index 00000000000..7f50e6c72fe --- /dev/null +++ b/vm/backend/demo/selftest/com/demo/SelfTest.java @@ -0,0 +1,1037 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.codename1.backend.Base64Url; +import com.codename1.backend.ByteSink; +import com.codename1.backend.Crypto; +import com.codename1.backend.Database; +import com.codename1.backend.Db; +import com.codename1.backend.DbPool; +import com.codename1.backend.Http; +import com.codename1.backend.Http1Date; +import com.codename1.backend.HttpServer; +import com.codename1.backend.Json; +import com.codename1.backend.Jwt; +import com.codename1.backend.ServerSocket; +import com.codename1.backend.Tcp; +import com.codename1.backend.Web; +import com.codename1.backend.aws.Credentials; +import com.codename1.backend.aws.S3; + +/** + * Unit tests for the backend runtime, run INSIDE a translated binary. + * + * They cannot be ordinary JUnit tests: every class here is backed by natives that + * only exist in a translated program, so running them on a JVM would test nothing + * that ships. The harness is deliberately tiny -- print a line per check, exit + * non-zero if any failed -- and a JUnit test builds this, runs it, and reads the + * result. + */ +public class SelfTest { + private static int passed; + private static final List failures = new ArrayList(); + + /** + * A saturated server must still answer a connection that arrives during the + * saturation. This looks like an exotic property and is not: a worker that + * keeps a busy keep-alive connection instead of handing it back makes the pool + * size the hard limit on concurrent clients, and the failure is invisible in + * every throughput number, because the connections that DO hold a worker are + * served at full speed while the rest wait forever. The bug this pins was found + * by a stray curl during a benchmark that was reporting 234k requests a second + * at the time. + * + * Two workers and four connections that never stop sending, so there is no + * arrangement in which a held connection is free to hold. + */ + private static void fairness() throws Exception { + HttpServer server = HttpServer.start("127.0.0.1", 0, 64, 2, new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + return HttpServer.Response.text(200, "ok"); + } + }); + final int port = server.getPort(); + final boolean[] stop = new boolean[1]; + Thread[] load = new Thread[4]; + try { + for(int t = 0 ; t < load.length ; t++) { + load[t] = new Thread(new Runnable() { + public void run() { + Tcp socket = null; + try { + socket = Tcp.connect("127.0.0.1", port, 2000); + byte[] req = ("GET /x HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n") + .getBytes("UTF-8"); + byte[] sink = new byte[1024]; + while(!stop[0]) { + socket.write(req, 0, req.length); + if(socket.read(sink, 0, sink.length) <= 0) { + return; + } + } + } catch (Exception ignored) { + // A loader that dies just reduces the pressure; the + // probe below is what decides the result. + } finally { + if(socket != null) { + socket.close(); + } + } + } + }); + load[t].start(); + } + Thread.sleep(300); + + // The probe runs on its own thread so a server that never answers fails + // the check instead of hanging the suite for the socket timeout. + final String[] answer = new String[1]; + Thread probe = new Thread(new Runnable() { + public void run() { + try { + answer[0] = new String(Http.get("127.0.0.1", port, "/probe").getBody(), "UTF-8"); + } catch (Exception err) { + answer[0] = "failed: " + err; + } + } + }); + probe.start(); + probe.join(5000); + check("a saturated server answers a new connection", "ok", + answer[0] == null ? "no response in 5s" : answer[0]); + } finally { + stop[0] = true; + for(int t = 0 ; t < load.length ; t++) { + load[t].join(2000); + } + server.stop(1000); + } + } + + /** + * The foreign-backed read buffer: a byte[] whose storage on the translated + * target is a C buffer the collector never allocated. + * + * The interesting assertion is the one after the GC cycles. An object outside + * every heap page is not swept, and gcMarkObject rejects a pointer that does + * not resolve, so it should come through untouched -- but "should" is the word + * that makes this worth a test, because the failure mode is silent corruption + * of a buffer every request reads through. + */ + private static void foreignBuffer() throws Exception { + byte[] buffer = ServerSocket.threadReadBuffer(4096); + check("a read buffer is provided", "true", String.valueOf(buffer != null)); + check("it is at least the size asked for", "true", + String.valueOf(buffer.length >= 4096)); + + // Behaves as an ordinary array: bounds, element access, arraycopy. + for(int iter = 0 ; iter < 4096 ; iter++) { + buffer[iter] = (byte)(iter & 0x7f); + } + byte[] copy = new byte[16]; + System.arraycopy(buffer, 100, copy, 0, 16); + check("arraycopy reads foreign storage", "100", String.valueOf(copy[0])); + boolean threw = false; + try { + int ignored = buffer[buffer.length]; + threw = ignored == -1 && false; + } catch (ArrayIndexOutOfBoundsException expected) { + threw = true; + } + check("bounds are enforced on it", "true", String.valueOf(threw)); + + // The same object, so a server reading through it allocates nothing. + check("the same buffer comes back", "true", + String.valueOf(ServerSocket.threadReadBuffer(4096) == buffer)); + + // Survive collections. Allocate enough to force real cycles, then check + // both the identity and every byte. + for(int round = 0 ; round < 3 ; round++) { + for(int iter = 0 ; iter < 20000 ; iter++) { + byte[] garbage = new byte[256]; + garbage[0] = (byte)iter; + } + System.gc(); + } + byte[] after = ServerSocket.threadReadBuffer(4096); + check("identity survives collection", "true", String.valueOf(after == buffer)); + int damaged = -1; + for(int iter = 0 ; iter < 4096 ; iter++) { + if(buffer[iter] != (byte)(iter & 0x7f)) { + damaged = iter; + break; + } + } + check("contents survive collection", "-1", String.valueOf(damaged)); + + // A grow keeps one Java identity and moves only the storage. + byte[] grown = ServerSocket.threadReadBuffer(65536); + check("a grow still yields a usable buffer", "true", + String.valueOf(grown != null && grown.length >= 65536)); + grown[65535] = 42; + check("the grown tail is writable", "42", String.valueOf(grown[65535])); + } + + public static void main(String[] args) throws Exception { + crypto(); + jwt(); + base64Url(); + json(); + httpDate(); + database(); + pool(); + foreignBuffer(); + fairness(); + web(); + clientTls(); + + System.out.println("passed=" + passed + " failed=" + failures.size()); + for(int iter = 0 ; iter < failures.size() ; iter++) { + System.out.println("FAIL " + failures.get(iter)); + } + System.out.println(failures.isEmpty() ? "SELFTEST OK" : "SELFTEST FAILED"); + if(!failures.isEmpty()) { + System.exit(1); + } + } + + // ------------------------------------------------------------------ + + private static void crypto() throws Exception { + // Known-answer tests, not round trips. A round trip passes just as happily + // against a wrong-but-consistent implementation. + check("sha256 known vector", + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + hex(Crypto.sha256(bytes("abc")))); + + // RFC 4231 test case 1. + byte[] key = new byte[20]; + for(int iter = 0 ; iter < key.length ; iter++) { + key[iter] = 0x0b; + } + check("hmac-sha256 RFC 4231 case 1", + "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", + hex(Crypto.hmacSha256(key, bytes("Hi There")))); + + byte[] a = Crypto.randomBytes(32); + byte[] b = Crypto.randomBytes(32); + check("randomBytes returns the requested length", "32", String.valueOf(a.length)); + check("randomBytes differ between calls", "true", String.valueOf(!hex(a).equals(hex(b)))); + + check("constantTime equal", "true", + String.valueOf(Crypto.equalsConstantTime(bytes("secret"), bytes("secret")))); + check("constantTime differing", "false", + String.valueOf(Crypto.equalsConstantTime(bytes("secret"), bytes("secret2")))); + check("constantTime same length, one bit apart", "false", + String.valueOf(Crypto.equalsConstantTime(bytes("secreta"), bytes("secretb")))); + check("constantTime null", "false", + String.valueOf(Crypto.equalsConstantTime(null, bytes("x")))); + + String stored = Crypto.hashPassword("hunter2"); + check("password hash is a pbkdf2 verifier", "true", + String.valueOf(stored.startsWith("pbkdf2$"))); + check("password hash is not the password", "true", + String.valueOf(stored.indexOf("hunter2") < 0)); + check("password verifies", "true", String.valueOf(Crypto.verifyPassword("hunter2", stored))); + check("wrong password rejected", "false", String.valueOf(Crypto.verifyPassword("hunter3", stored))); + check("empty password rejected", "false", String.valueOf(Crypto.verifyPassword("", stored))); + // A null password must not become an empty-password account: utf8(null) is + // an empty array, so hashing one produced a verifier that "" satisfies. + // Asserted on BOTH runtimes, because the two have separate Crypto arms. + String nullOutcome; + try { + Crypto.hashPassword(null); + nullOutcome = "accepted"; + } catch (IllegalArgumentException refused) { + nullOutcome = "refused"; + } + check("a null password is refused", "refused", nullOutcome); + // A stored row with empty salt and hash decoded to two EMPTY arrays, not + // nulls, so the null check let it through, pbkdf2 derived zero bytes and + // comparing empty with empty was true: that row accepted every password. + check("a degenerate stored hash accepts nothing", "false", + String.valueOf(Crypto.verifyPassword("anything", "pbkdf2$1$$"))); + check("a short salt is refused too", "false", + String.valueOf(Crypto.verifyPassword("anything", "pbkdf2$1$AA$AA"))); + // Two hashes of one password must differ, or the salt is not being used. + check("hashes are salted", "true", + String.valueOf(!stored.equals(Crypto.hashPassword("hunter2")))); + check("malformed stored value rejected", "false", + String.valueOf(Crypto.verifyPassword("hunter2", "not-a-hash"))); + check("truncated stored value rejected", "false", + String.valueOf(Crypto.verifyPassword("hunter2", "pbkdf2$1000$abc"))); + } + + private static void jwt() throws Exception { + byte[] secret = Crypto.randomBytes(32); + Map claims = new LinkedHashMap(); + claims.put("sub", "shai"); + String token = Jwt.issue(claims, secret, 60); + // JavaAPI's String has no split(); count the separators instead. + check("token has two dots", "2", String.valueOf(countChar(token, '.'))); + + Map verified = Jwt.verify(token, secret); + // issue() has always refused a secret under 32 bytes as forgeable; verify() + // took any. A deployment configured with an empty one would have accepted a + // signature anybody could compute over claims of their choosing, so the + // dangerous half was the one that failed open. + boolean refusedShortSecret = false; + try { + Jwt.verify(token, new byte[0]); + } catch (Exception expected) { + refusedShortSecret = true; + } + check("verifying with a short secret is refused", "true", + String.valueOf(refusedShortSecret)); + check("subject survives", "shai", String.valueOf(verified.get("sub"))); + check("expiry is set", "true", String.valueOf(verified.get("exp") instanceof Number)); + + checkThrows("a tampered signature is rejected", new Body() { + public void run() throws Exception { + byte[] s = Crypto.randomBytes(32); + Map c = new LinkedHashMap(); + c.put("sub", "shai"); + String t = Jwt.issue(c, s, 60); + // Flip a character in the MIDDLE of the signature. Flipping the + // last one used to decode to identical bytes, because the trailing + // bits of a base64 group are padding -- which is why the decoder + // now rejects a non-canonical encoding. + int at = t.length() - 10; + char replacement = t.charAt(at) == 'A' ? 'B' : 'A'; + Jwt.verify(t.substring(0, at) + replacement + t.substring(at + 1), s); + } + }); + checkThrows("a token signed with another key is rejected", new Body() { + public void run() throws Exception { + Map c = new LinkedHashMap(); + c.put("sub", "shai"); + String t = Jwt.issue(c, Crypto.randomBytes(32), 60); + Jwt.verify(t, Crypto.randomBytes(32)); + } + }); + checkThrows("an expired token is rejected", new Body() { + public void run() throws Exception { + byte[] s = Crypto.randomBytes(32); + Map c = new LinkedHashMap(); + c.put("sub", "shai"); + // Negative lifetime: issued already expired. + Jwt.verify(Jwt.issue(c, s, -60), s); + } + }); + checkThrows("a short signing secret is refused", new Body() { + public void run() throws Exception { + Jwt.issue(new LinkedHashMap(), new byte[16], 60); + } + }); + checkThrows("alg=none is rejected", new Body() { + public void run() throws Exception { + // The classic JWT hole: a verifier that reads its algorithm out of + // the token it is checking accepts this. + String header = Base64Url.encode(bytes("{\"alg\":\"none\",\"typ\":\"JWT\"}")); + String payload = Base64Url.encode(bytes("{\"sub\":\"attacker\",\"exp\":9999999999}")); + Jwt.verify(header + "." + payload + ".", Crypto.randomBytes(32)); + } + }); + checkThrows("a malformed token is rejected", new Body() { + public void run() throws Exception { + Jwt.verify("not.a.token", Crypto.randomBytes(32)); + } + }); + + check("bearer is extracted", "abc", String.valueOf(Jwt.bearer("Bearer abc"))); + check("bearer is case-insensitive", "abc", String.valueOf(Jwt.bearer("bearer abc"))); + check("a non-bearer header yields null", "null", String.valueOf(Jwt.bearer("Basic abc"))); + check("a null header yields null", "null", String.valueOf(Jwt.bearer(null))); + } + + private static void base64Url() throws Exception { + // Padding has to be contiguous and at the end, and the bits it stands for + // have to be zero. "AA=A" satisfied "final group, '=' at index 2" and then + // took the 'A' after it as data, returning three bytes for a string no + // encoder can produce; "AB==" gave a second spelling of a byte "AA==" + // already spells, which a strict decoder must not accept. + check("padding followed by data is refused", "true", + String.valueOf(com.codename1.backend.Base64.decode("AA=A") == null)); + check("nonzero padding bits are refused", "true", + String.valueOf(com.codename1.backend.Base64.decode("AB==") == null)); + check("ordinary padding still decodes", "1", + String.valueOf(com.codename1.backend.Base64.decode("AA==").length)); + check("encodes without padding", "SGVsbG8", Base64Url.encode(bytes("Hello"))); + check("one leftover byte", "SGU", Base64Url.encode(bytes("He"))); + check("two leftover bytes", "SGVs", Base64Url.encode(bytes("Hel"))); + check("round trips", "Hello, world", + new String(Base64Url.decode(Base64Url.encode(bytes("Hello, world"))), "UTF-8")); + // The url alphabet: '-' and '_' where standard base64 has '+' and '/'. + byte[] high = new byte[]{(byte)0xfb, (byte)0xff, (byte)0xbf}; + check("uses the url alphabet", "true", + String.valueOf(Base64Url.encode(high).indexOf('+') < 0 + && Base64Url.encode(high).indexOf('/') < 0)); + check("decodes the url alphabet", "fbffbf", hex(Base64Url.decode(Base64Url.encode(high)))); + // A base64 group is 2, 3 or 4 characters; a single leftover encodes nothing. + check("rejects a lone trailing character", "null", + String.valueOf(Base64Url.decode("SGVsbG8AA"))); + check("rejects a character outside the alphabet", "null", + String.valueOf(Base64Url.decode("SGVs*G8"))); + check("empty round trips", "0", String.valueOf(Base64Url.decode("").length)); + // Non-canonical: "SGVsbG9" leaves bits set that a canonical encoder would + // have left zero, so several strings would decode alike. + check("rejects a non-canonical encoding", "null", + String.valueOf(Base64Url.decode("SGVsbG9"))); + check("accepts the canonical form of the same bytes", "5", + String.valueOf(Base64Url.decode("SGVsbG8").length)); + } + + /** + * The two JSON writers have to answer the same bytes. HTTP/1.1 writes through + * the ByteSink one and HTTP/2 through the String one, so a disagreement means + * one handler returns two different documents depending on which protocol the + * client negotiated -- and nothing in either path would ever notice. Float + * was the second such defect (byte[] was the first), which is why this + * compares the writers rather than either one's output. + */ + private static void bothJsonWritersAgree() throws Exception { + Object[] values = new Object[] { + Float.valueOf(1.2f), Float.valueOf(-0.1f), Float.valueOf(3.4e38f), + Double.valueOf(1.2d), Double.valueOf(1e300), Long.valueOf(9007199254740993L), + Integer.valueOf(-7), Boolean.TRUE, "text", new byte[] {1, 2, 3}, + }; + for(int iter = 0 ; iter < values.length ; iter++) { + ByteSink sink = new ByteSink(64); + Json.write(values[iter], sink); + String viaSink = new String(sink.bytes(), 0, sink.length(), "UTF-8"); + check("both JSON writers agree on " + values[iter].getClass().getName(), + Json.write(values[iter]), viaSink); + } + // And the float keeps its OWN spelling rather than the double it widens to. + check("a float is not widened", "1.2", Json.write(Float.valueOf(1.2f))); + } + + /** + * A malformed HTTP date must be NO date. Every field is read at a fixed + * offset and handed to a civil-date routine that normalises whatever it is + * given, so "99 Nov 9999 99:99:99" became a date far in the future and a + * conditional request read it as newer than the file -- answering 304, with + * no content, to a client that had nothing cached. + */ + private static void malformedDatesAreNotDates() throws Exception { + String[] bad = new String[] { + "Sun, 99 Nov 9999 99:99:99 BAD", // out of range, wrong suffix + "Sun, 06 Nov 1994 08:49:37 UTC", // IMF-fixdate is GMT + "Sun, 06 Nov 1994 08:49:37", // truncated + "Sun, 31 Feb 1994 08:49:37 GMT", // a day that does not exist + "Sun, 06 Nov 1994 25:00:00 GMT", // hour out of range + "Sunday, 06-Nov-94 08:49:37 GMT", // RFC 850, deliberately unsupported + }; + for(int iter = 0 ; iter < bad.length ; iter++) { + check("a malformed date is refused: " + bad[iter], "-1", + String.valueOf(Http1Date.parse(bad[iter]))); + } + // And the one real form still parses, round-tripping through the writer. + long when = Http1Date.parse("Sun, 06 Nov 1994 08:49:37 GMT"); + check("a valid IMF-fixdate parses", "784111777000", String.valueOf(when)); + check("and formats back", "Sun, 06 Nov 1994 08:49:37 GMT", Http1Date.format(when)); + } + + /** + * Protocol tokens are folded by hand, never with String.toLowerCase(), which + * is locale sensitive and has no root-locale overload here. The values below + * all contain an I, which is the character a Turkish locale folds to a + * dotless i -- so a lookup keyed on the folded form stops matching and the + * header, the extension or the scheme reads as absent with nothing thrown. + */ + private static void asciiFoldingIsLocaleIndependent() throws Exception { + // Jwt.bearer is the one such fold reachable from here; StaticFiles' + // content type and the Web arms' header index are package-private, and + // BackendHttpIntegrationTest exercises those over the wire instead. + check("an upper-case bearer scheme is still a bearer scheme", + "abc.def.ghi", String.valueOf(Jwt.bearer("BEARER abc.def.ghi"))); + check("a mixed-case one too", + "abc.def.ghi", String.valueOf(Jwt.bearer("Bearer abc.def.ghi"))); + check("and the lower-case spelling is unchanged", + "abc.def.ghi", String.valueOf(Jwt.bearer("bearer abc.def.ghi"))); + check("something that is not a bearer header is still refused", + "null", String.valueOf(Jwt.bearer("Basic abc"))); + } + + /** + * PATCH is where the two runtimes genuinely differ, and this says so rather + * than pretending otherwise: the packaged one sends it, while Java SE's + * HttpURLConnection refuses the verb outright on every JDK measured -- 8, 21 + * and 25 -- and the reflection trick usually reached for works only on 8. + * + * What BOTH must satisfy is that the developer is never left holding an + * unexplained failure. Packaged, the request is attempted; locally, it fails + * with a message that names the limitation and says the packaged binary can + * do it. The JDK's own "Invalid HTTP method: PATCH" says none of that, and + * that opaque failure is what this asserts is gone. + */ + private static void patchIsASendableVerb() throws Exception { + String outcome; + try { + Web.Result r = Web.request("PATCH", "http://127.0.0.1:1/nothing", null, null); + // Nothing is listening, so a failed CONNECTION is the expected answer + // where the verb IS sendable. What matters is that the verb was not + // what stopped it. + outcome = r == null || r.getStatus() <= 0 ? "sent or explained" : "answered"; + } catch (Exception err) { + String message = String.valueOf(err.getMessage()); + // Either it went out and the connection failed, or it was refused with + // the explanation. Anything else is the opaque JDK error. + outcome = message.indexOf("cannot send") >= 0 + || message.indexOf("Connection refused") >= 0 + || message.indexOf("failed") >= 0 + ? "sent or explained" : "opaque: " + message; + } + check("PATCH is sent, or refused with a reason", "sent or explained", outcome); + } + + /** + * A repeated outbound header must survive on BOTH arms. + * + * libcurl appends every list entry it is given, so two Cookie lines both go + * out of the packaged binary. HttpURLConnection's setRequestProperty REPLACES, + * so the local arm sent only the last one -- an integration that depends on a + * repeatable header worked once packaged and quietly sent half of what it + * meant to under cn1:backend, which is the worst way round for a dev loop to + * be wrong. Echoed back by a server here rather than inspected, because the + * two arms have no shared way to ask what they sent. + */ + private static void repeatedOutboundHeadersSurvive() throws Exception { + HttpServer server = HttpServer.start("127.0.0.1", 0, 16, 1, new HttpServer.Handler() { + public HttpServer.Response handle(HttpServer.Request request) { + String seen = request.getHeader("x-repeat"); + return HttpServer.Response.text(200, seen == null ? "absent" : seen); + } + }); + try { + List headers = new ArrayList(); + headers.add("X-Repeat: one"); + headers.add("X-Repeat: two"); + Web.Result r = Web.request("GET", + "http://127.0.0.1:" + server.getPort() + "/", headers, null); + String body = r == null ? "null" : r.getBodyAsString(); + // The server joins repeats with ", " (RFC 9110 5.3), so BOTH values + // are present exactly when both lines were sent. Asserting on the + // joined string rather than on a count keeps this true whichever + // order the arms emit them in. + check("a repeated request header keeps its first value", "true", + String.valueOf(body != null && body.indexOf("one") >= 0)); + check("a repeated request header keeps its second value", "true", + String.valueOf(body != null && body.indexOf("two") >= 0)); + } finally { + server.stop(); + } + } + + /** + * A negative connectTimeout is refused by the PARSER, so both arms fail the + * same way. Left to the arms, Java SE threw IllegalArgumentException out of + * Socket.connect while the packaged client read any non-positive value as + * "block forever" and waited out the OS TCP timeout: the same URL, an error + * on one side and a hang on the other. + */ + private static void negativeConnectTimeoutsAreRefused() throws Exception { + String outcome; + try { + // A NETWORK url: the query string is only parsed for the engines + // that have a connection to time out, and sqlite has none. + Database.open("postgres://u:p@127.0.0.1:1/db?connectTimeout=-1"); + outcome = "accepted"; + } catch (Exception refused) { + String message = String.valueOf(refused.getMessage()); + // The PARSER's wording specifically. Matching on "negative" alone + // also matches the JDK's own "timeout can't be negative" out of + // Socket.connect -- which is the arm-specific failure this check + // exists to replace, so it would have passed either way. + outcome = message.indexOf("must not be negative") >= 0 + ? "refused" : "other: " + message; + } + check("a negative connectTimeout is refused", "refused", outcome); + + // And at the API below the URL parser, where a caller can reach it + // directly. The two arms disagreed: Java SE fails immediately out of + // Socket.connect while the packaged native reads every non-positive value + // as "block with no deadline", so the same call hangs for the OS TCP + // timeout once packaged. Zero keeps its documented meaning. + String tcpOutcome; + try { + Tcp.connect("127.0.0.1", 1, -1); + tcpOutcome = "accepted"; + } catch (IllegalArgumentException refused) { + tcpOutcome = "refused"; + } catch (Exception other) { + tcpOutcome = "other: " + other; + } + check("a negative TCP connect timeout is refused", "refused", tcpOutcome); + } + + private static void json() throws Exception { + bothJsonWritersAgree(); + patchIsASendableVerb(); + repeatedOutboundHeadersSurvive(); + negativeConnectTimeoutsAreRefused(); + malformedDatesAreNotDates(); + asciiFoldingIsLocaleIndependent(); + Map parsed = Json.parseObject("{\"a\":1,\"b\":\"two\",\"c\":true,\"d\":null,\"e\":1.5}"); + // Integers must stay integers: a long round-tripped through double loses + // precision above 2^53, and ids are exactly the values that get large. + check("integers parse as Long", "true", String.valueOf(parsed.get("a") instanceof Long)); + check("reals parse as Double", "true", String.valueOf(parsed.get("e") instanceof Double)); + check("strings parse", "two", String.valueOf(parsed.get("b"))); + check("booleans parse", "true", String.valueOf(parsed.get("c"))); + check("nulls parse", "null", String.valueOf(parsed.get("d"))); + + check("large integers keep precision", "9007199254740993", + String.valueOf(Json.parseObject("{\"n\":9007199254740993}").get("n"))); + + Map nested = Json.parseObject("{\"o\":{\"p\":[1,2,{\"q\":\"r\"}]}}"); + Map inner = (Map)nested.get("o"); + List list = (List)inner.get("p"); + check("nesting survives", "3", String.valueOf(list.size())); + check("objects inside arrays survive", "r", + String.valueOf(((Map)list.get(2)).get("q"))); + + check("escapes decode", "a\"b\\c\nd", + String.valueOf(Json.parseObject("{\"s\":\"a\\\"b\\\\c\\nd\"}").get("s"))); + check("unicode escapes decode", "\u00e9", + String.valueOf(Json.parseObject("{\"s\":\"\\u00e9\"}").get("s"))); + + // RFC 8259 requires anything below U+0020 to arrive escaped. Accepting a + // literal one let this parser and whatever validates upstream disagree + // about where the string ended. + boolean refusedControl = false; + try { + Json.parseObject("{\"s\":\"a\nb\"}"); + } catch (Exception expected) { + refusedControl = true; + } + check("a raw control character in a string is refused", "true", + String.valueOf(refusedControl)); + + // A bucket name is interpolated into the request URL, so one carrying a + // slash steers it: into the path here, and into the HOST -- taking the + // signed access key id and session token to a server of the caller's + // choosing -- under virtual-hosted addressing. presign computes locally + // and sends nothing, which is what makes this checkable here. + // + // forEndpoint is path style, so this covers that guard. The virtual-hosted + // one needs forRegion, which resolves real credentials, so it is not + // reachable from a self-test that must run with none. + S3 s3 = S3.forEndpoint(new Credentials("AKIDEXAMPLE", "secret", null), + "us-east-1", "s3.us-east-1.amazonaws.com"); + boolean refusedBucket = false; + try { + s3.presignGet("attacker.example/ignored", "k", 60); + } catch (Exception expected) { + refusedBucket = true; + } + // Both arms must refuse the same configuration. The translated one used to + // cast this to an unsigned short, so 65536 became 0 and the server came up + // on an arbitrary port while the Java SE loop rejected it. + boolean refusedPort = false; + try { + ServerSocket.bind(null, 65536, 16); + } catch (Exception expected) { + refusedPort = true; + } + check("a port above 65535 is refused", "true", String.valueOf(refusedPort)); + check("a bucket name that rewrites the host is refused", "true", + String.valueOf(refusedBucket)); + boolean signedOrdinary = false; + try { + signedOrdinary = s3.presignGet("ordinary-bucket", "k", 60) + .indexOf("/ordinary-bucket/k") > 0; + } catch (Exception err) { + signedOrdinary = false; + } + check("an ordinary bucket still signs", "true", String.valueOf(signedOrdinary)); + // A lifetime outside SigV4's 1 second to 7 days produces a URL that looks + // right and is refused when the device tries to use it, which is a failure + // a long way from the call that caused it. + boolean refusedLifetime = false; + try { + s3.presignGet("ordinary-bucket", "k", 0); + } catch (Exception expected) { + refusedLifetime = true; + } + check("a presigned URL with no lifetime is refused", "true", + String.valueOf(refusedLifetime)); + check("the same character escaped is accepted", "a\nb", + String.valueOf(Json.parseObject("{\"s\":\"a\\nb\"}").get("s"))); + + Map out = new LinkedHashMap(); + out.put("q", "a\"b"); + out.put("n", new Long(5)); + check("writing escapes quotes", "{\"q\":\"a\\\"b\",\"n\":5}", Json.write(out)); + check("writing a control character escapes it", "{\"q\":\"a\\nb\"}", + Json.write(single("q", "a\nb"))); + // NaN and Infinity have no JSON form; emitting them produces a document no + // parser will read back. + check("NaN is written as null", "{\"q\":null}", + Json.write(single("q", new Double(Double.NaN)))); + + checkThrows("trailing content is rejected", new Body() { + public void run() throws Exception { + Json.parse("{\"a\":1} junk"); + } + }); + checkThrows("an unterminated string is rejected", new Body() { + public void run() throws Exception { + Json.parse("{\"a\":\"oops}"); + } + }); + checkThrows("a missing value is rejected", new Body() { + public void run() throws Exception { + Json.parse("{\"a\":}"); + } + }); + checkThrows("an array asked for as an object is rejected", new Body() { + public void run() throws Exception { + Json.parseObject("[1,2]"); + } + }); + } + + private static void httpDate() throws Exception { + // The example from the HTTP specification itself. + check("formats the RFC example", "Sun, 06 Nov 1994 08:49:37 GMT", + Http1Date.format(784111777000L)); + check("parses the RFC example", "784111777000", + String.valueOf(Http1Date.parse("Sun, 06 Nov 1994 08:49:37 GMT"))); + check("epoch formats", "Thu, 01 Jan 1970 00:00:00 GMT", Http1Date.format(0)); + check("a leap day survives a round trip", "Sat, 29 Feb 2020 12:00:00 GMT", + Http1Date.format(Http1Date.parse("Sat, 29 Feb 2020 12:00:00 GMT"))); + check("garbage yields -1", "-1", String.valueOf(Http1Date.parse("not a date"))); + check("null yields -1", "-1", String.valueOf(Http1Date.parse(null))); + } + + private static void database() throws Exception { + Db db = Db.open(":memory:"); + try { + db.execute("CREATE TABLE t (id INTEGER PRIMARY KEY AUTOINCREMENT, " + + "name TEXT, weight REAL, data BLOB, maybe TEXT)", null); + db.execute("INSERT INTO t (name, weight, data, maybe) VALUES (?, ?, ?, ?)", + new Object[]{"first", new Double(1.5), bytes("blob-bytes"), null}); + check("lastInsertId", "1", String.valueOf(db.lastInsertId())); + + List rows = db.query("SELECT id, name, weight, data, maybe FROM t", null); + check("one row", "1", String.valueOf(rows.size())); + Map row = (Map)rows.get(0); + check("integer column is Long", "true", String.valueOf(row.get("id") instanceof Long)); + check("real column is Double", "true", String.valueOf(row.get("weight") instanceof Double)); + check("text column is String", "first", String.valueOf(row.get("name"))); + check("blob column is byte[]", "true", String.valueOf(row.get("data") instanceof byte[])); + check("blob round trips", "blob-bytes", new String((byte[])row.get("data"), "UTF-8")); + check("null column is null", "null", String.valueOf(row.get("maybe"))); + + // The reason parameters are bound and never interpolated. + db.execute("INSERT INTO t (name) VALUES (?)", + new Object[]{"bobby'); DROP TABLE t; --"}); + check("an injection attempt is stored as data", + "2", String.valueOf(db.query("SELECT id FROM t", null).size())); + + check("changes are counted", "2", + String.valueOf(db.execute("UPDATE t SET weight = 9.0", null))); + + // A transaction that throws must leave nothing behind. + int before = db.query("SELECT id FROM t", null).size(); + boolean threw = false; + try { + db.transaction(new Db.Work() { + public Object run(Db conn) throws Exception { + conn.execute("INSERT INTO t (name) VALUES (?)", new Object[]{"doomed"}); + throw new IllegalStateException("deliberate"); + } + }); + } catch (IllegalStateException err) { + threw = true; + } + check("the transaction body's failure propagates", "true", String.valueOf(threw)); + check("the failed transaction rolled back", String.valueOf(before), + String.valueOf(db.query("SELECT id FROM t", null).size())); + + // And one that returns must commit. + Object result = db.transaction(new Db.Work() { + public Object run(Db conn) throws Exception { + conn.execute("INSERT INTO t (name) VALUES (?)", new Object[]{"kept"}); + return "done"; + } + }); + check("the transaction returns its value", "done", String.valueOf(result)); + check("the committed row is there", String.valueOf(before + 1), + String.valueOf(db.query("SELECT id FROM t", null).size())); + + checkThrows("bad SQL is reported", new Body() { + public void run() throws Exception { + Db d = Db.open(":memory:"); + try { + d.execute("SELECT * FROM no_such_table", null); + } finally { + d.close(); + } + } + }); + } finally { + db.close(); + } + + checkThrows("using a closed database is refused", new Body() { + public void run() throws Exception { + Db d = Db.open(":memory:"); + d.close(); + d.query("SELECT 1", null); + } + }); + } + + private static void pool() throws Exception { + checkThrows("an in-memory database cannot be pooled", new Body() { + public void run() throws Exception { + // Each connection would get its own private database, so every + // caller would see a different one. + DbPool.open(":memory:", 2, 1000); + } + }); + + String path = System.getenv("CN1_SELFTEST_DB"); + if(path == null) { + note("pool concurrency skipped: set CN1_SELFTEST_DB to a writable path"); + return; + } + final DbPool pool = DbPool.open(path, 4, 5000); + try { + Db setup = pool.borrow(); + setup.execute("DROP TABLE IF EXISTS counter", null); + setup.execute("CREATE TABLE counter (id INTEGER PRIMARY KEY AUTOINCREMENT, who TEXT)", null); + pool.release(setup); + + final int threads = 8; + final int each = 25; + final int[] failed = new int[1]; + Thread[] workers = new Thread[threads]; + for(int t = 0 ; t < threads ; t++) { + final String who = "worker-" + t; + workers[t] = new Thread(new Runnable() { + public void run() { + for(int i = 0 ; i < each ; i++) { + try { + pool.inTransaction(new Db.Work() { + public Object run(Db db) throws Exception { + db.execute("INSERT INTO counter (who) VALUES (?)", + new Object[]{who}); + return null; + } + }); + } catch (Exception err) { + synchronized(failed) { + failed[0]++; + } + } + } + } + }); + workers[t].start(); + } + for(int t = 0 ; t < threads ; t++) { + workers[t].join(); + } + Db check = pool.borrow(); + List rows = check.query("SELECT COUNT(*) AS c FROM counter", null); + long count = ((Number)((Map)rows.get(0)).get("c")).longValue(); + List distinct = check.query("SELECT COUNT(DISTINCT who) AS c FROM counter", null); + long writers = ((Number)((Map)distinct.get(0)).get("c")).longValue(); + pool.release(check); + check("every pooled write landed", String.valueOf(threads * each), String.valueOf(count)); + check("every worker got a connection", String.valueOf(threads), String.valueOf(writers)); + check("no pooled transaction failed", "0", String.valueOf(failed[0])); + } finally { + pool.close(); + } + } + + private static void web() throws Exception { + checkThrows("a null URL is refused", new Body() { + public void run() throws Exception { + Web.get(null); + } + }); + checkThrows("an unresolvable host fails rather than returning a status", new Body() { + public void run() throws Exception { + Web.get("https://this-host-does-not-exist.invalid/"); + } + }); + if(System.getenv("CN1_SELFTEST_NETWORK") == null) { + note("network checks skipped: set CN1_SELFTEST_NETWORK=1 to run them"); + return; + } + Web.Result ok = Web.get("https://api.github.com/zen"); + check("an https GET succeeds", "true", String.valueOf(ok.isSuccess())); + check("the body arrives", "true", String.valueOf(ok.getBodyAsString().length() > 0)); + checkThrows("an expired certificate is rejected", new Body() { + public void run() throws Exception { + // Verification being ON is the whole reason to use a TLS library. + Web.get("https://expired.badssl.com/"); + } + }); + } + + // ------------------------------------------------------------------ + + /** + * Outbound TLS as an upgrade of a connected socket -- the shape the database + * clients need, and the one Web.get (libcurl on the native target) does not + * exercise. + * + * Both halves matter. The positive check proves the handshake completes and + * bytes flow; the negative one proves the certificate is actually VERIFIED, + * which is the part that fails open. An unverified TLS connection looks + * exactly like a verified one until someone is in the middle. + */ + private static void clientTls() throws Exception { + if(System.getenv("CN1_SELFTEST_NETWORK") == null) { + note("outbound TLS checks skipped: set CN1_SELFTEST_NETWORK=1 to run them"); + return; + } + Tcp plain = Tcp.connect("api.github.com", 443, 10000); + try { + check("a fresh socket is not secure", "false", String.valueOf(plain.isSecure())); + plain.startTls("api.github.com"); + check("the socket is secure after the upgrade", "true", + String.valueOf(plain.isSecure())); + byte[] request = bytes("GET /zen HTTP/1.0\r\nHost: api.github.com\r\n" + + "User-Agent: cn1-backend-selftest\r\nConnection: close\r\n\r\n"); + plain.write(request, 0, request.length); + String response = readAll(plain); + check("the encrypted response is an HTTP one", "true", + String.valueOf(response.startsWith("HTTP/1."))); + } finally { + plain.close(); + } + + // The name on the certificate has to be checked, not just its chain. This + // connects to a host that HAS a valid certificate and asks for a different + // name, so only the name check can reject it. + final Tcp mismatched = Tcp.connect("api.github.com", 443, 10000); + try { + checkThrows("a certificate for another host is rejected", new Body() { + public void run() throws Exception { + mismatched.startTls("example.invalid"); + } + }); + } finally { + mismatched.close(); + } + + final Tcp expired = Tcp.connect("expired.badssl.com", 443, 10000); + try { + checkThrows("an expired certificate is rejected on an upgraded socket", new Body() { + public void run() throws Exception { + expired.startTls("expired.badssl.com"); + } + }); + } finally { + expired.close(); + } + } + + /** Reads to end of stream. Only used on the small self-test responses. */ + private static String readAll(Tcp connection) throws Exception { + StringBuilder out = new StringBuilder(); + byte[] buffer = new byte[4096]; + while(true) { + int n = connection.read(buffer, 0, buffer.length); + if(n <= 0) { + return out.toString(); + } + out.append(new String(buffer, 0, n, "UTF-8")); + } + } + + private interface Body { + void run() throws Exception; + } + + private static void check(String name, String expected, String actual) { + if(expected.equals(actual)) { + passed++; + } else { + failures.add(name + ": expected <" + expected + "> but was <" + actual + ">"); + } + } + + private static void checkThrows(String name, Body body) { + try { + body.run(); + failures.add(name + ": expected an exception, none was thrown"); + } catch (Exception expected) { + passed++; + } + } + + private static void note(String message) { + System.out.println("NOTE " + message); + } + + private static Map single(String key, Object value) { + Map out = new LinkedHashMap(); + out.put(key, value); + return out; + } + + private static byte[] bytes(String value) { + try { + return value.getBytes("UTF-8"); + } catch (Exception err) { + return new byte[0]; + } + } + + private static int countChar(String value, char c) { + int count = 0; + for(int iter = 0 ; iter < value.length() ; iter++) { + if(value.charAt(iter) == c) { + count++; + } + } + return count; + } + + private static String hex(byte[] data) { + if(data == null) { + return "null"; + } + StringBuilder out = new StringBuilder(); + for(int iter = 0 ; iter < data.length ; iter++) { + int v = data[iter] & 0xff; + out.append("0123456789abcdef".charAt(v >>> 4)); + out.append("0123456789abcdef".charAt(v & 15)); + } + return out.toString(); + } +} diff --git a/vm/backend/demo/uncaught/com/demo/Uncaught.java b/vm/backend/demo/uncaught/com/demo/Uncaught.java new file mode 100644 index 00000000000..dbd56fa20c4 --- /dev/null +++ b/vm/backend/demo/uncaught/com/demo/Uncaught.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.io.IOException; + +/** + * An exception that nothing catches, thrown from inside a catch block. + * + * This is the shape that used to be silently discarded on the clean target: + * throwException walked the try-block stack, found no handler, and RETURNED, so + * the generated code carried on with the statement after the throw. A server then + * kept running with whatever half-built state the failed operation left behind -- + * in the case that found this, a null database handle that segfaulted two + * statements later. + * + * Driven by BackendUncaughtExceptionTest, which requires the message, a stack + * trace and a non-zero exit; the marker below must NOT be printed. + */ +public class Uncaught { + public static void main(String[] args) throws Exception { + System.out.println("before the throw"); + try { + open(); + } catch (IOException err) { + // Rethrowing from a catch, out of a main that has no other handler. + throw err; + } + } + + private static void open() throws IOException { + try { + throw new IOException("deliberate failure with a message"); + } catch (IOException err) { + throw err; + } + } +} diff --git a/vm/backend/demo/webcheck/com/demo/WebCheck.java b/vm/backend/demo/webcheck/com/demo/WebCheck.java new file mode 100644 index 00000000000..28b813b7ec8 --- /dev/null +++ b/vm/backend/demo/webcheck/com/demo/WebCheck.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import java.util.ArrayList; +import java.util.List; + +import com.codename1.backend.Web; + +/** + * Talks to a running backend using the PACKAGED outbound client, and reports what + * came back. + * + * Every other test of this server drives it from JUnit over a raw socket or an + * HttpURLConnection, so the client half of the packaged runtime -- Web, and the + * libcurl behind it -- was only ever exercised against dead ports and mocks. Two + * things follow from testing it this way instead. The verbs are proved end to + * end: PATCH is the one Java SE cannot send at all, so it can be verified HERE + * and nowhere else. And the two halves meet over a real socket, which is the + * only place a disagreement between them can actually show up. + * + * CN1_WEBCHECK_BASE names the server, for example http://127.0.0.1:8080. + */ +public class WebCheck { + private static int passed; + private static final List failures = new ArrayList(); + + public static void main(String[] args) throws Exception { + String base = System.getenv("CN1_WEBCHECK_BASE"); + if(base == null || base.length() == 0) { + System.out.println("CN1_WEBCHECK_BASE is not set"); + System.out.println("WEBCHECK FAILED"); + System.exit(1); + } + + // The server echoes the method it saw, so this compares what ARRIVED + // against what was asked for rather than trusting the client's own idea. + check("GET arrives as GET", "method=GET len=0", methodSeenBy(base, "GET")); + check("POST arrives as POST", "method=POST len=0", methodSeenBy(base, "POST")); + check("PUT arrives as PUT", "method=PUT len=0", methodSeenBy(base, "PUT")); + check("DELETE arrives as DELETE", "method=DELETE len=0", methodSeenBy(base, "DELETE")); + // The one the local runtime refuses outright. If the packaged client ever + // stops sending it, this is the only test that would notice. + check("PATCH arrives as PATCH", "method=PATCH len=0", methodSeenBy(base, "PATCH")); + + // A body, over a real socket, measured by the server rather than by the + // client -- so what is proved is that the bytes ARRIVED, not that they + // were handed to the transport. + Web.Result posted = Web.request("POST", base + "/echo", + header("Content-Type: application/json"), utf8("{\"a\":1}")); + check("a body arrives whole", "method=POST len=7", + posted == null ? "no result" : posted.getBodyAsString()); + + // And one big enough to cross the server's buffer growth several times. + StringBuilder big = new StringBuilder(); + for(int iter = 0 ; iter < 100000 ; iter++) { + big.append('x'); + } + Web.Result large = Web.request("POST", base + "/echo", null, utf8(big.toString())); + check("a large body arrives whole", "method=POST len=100000", + large == null ? "no result" : large.getBodyAsString()); + + // A response header the client must be able to read back. + Web.Result health = Web.request("GET", base + "/healthz", null, null); + check("a response carries its content type", "true", + String.valueOf(health != null + && health.getHeader("content-type") != null)); + + System.out.println("passed=" + passed + " failed=" + failures.size()); + for(int iter = 0 ; iter < failures.size() ; iter++) { + System.out.println("FAIL " + failures.get(iter)); + } + System.out.println(failures.isEmpty() ? "WEBCHECK OK" : "WEBCHECK FAILED"); + if(!failures.isEmpty()) { + System.exit(1); + } + } + + /** What the server says it received, or the transport error that stopped it. */ + private static String methodSeenBy(String base, String method) { + try { + Web.Result r = Web.request(method, base + "/echo", null, null); + if(r == null) { + return "no result"; + } + if(r.getStatus() != 200) { + return "status " + r.getStatus() + " " + r.getError(); + } + return r.getBodyAsString(); + } catch (Exception err) { + return "threw " + err.getMessage(); + } + } + + private static List header(String line) { + List out = new ArrayList(); + out.add(line); + return out; + } + + private static byte[] utf8(String value) throws Exception { + return value.getBytes("UTF-8"); + } + + private static void check(String name, String expected, String actual) { + if(expected.equals(actual)) { + passed++; + } else { + failures.add(name + ": expected <" + expected + "> but was <" + actual + ">"); + } + } +} diff --git a/vm/backend/docker/Containerfile.glibc b/vm/backend/docker/Containerfile.glibc new file mode 100644 index 00000000000..6af2daa6955 --- /dev/null +++ b/vm/backend/docker/Containerfile.glibc @@ -0,0 +1,22 @@ +# Builder image for glibc Linux backend binaries. +# +# The counterpart to Containerfile.musl. A static musl binary runs in a scratch +# image and depends on nothing; a glibc binary is what belongs in a base image +# that already carries libc and OpenSSL and patches them on its own schedule -- +# which is the arrangement most organisations have a policy about. Both are one +# clang invocation over the same generated C. +# +# Debian's own libcurl, OpenSSL and nghttp2 are used rather than a build from +# source: the whole point of this target is to link against what the distribution +# ships and updates. +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + clang libcurl4-openssl-dev libssl-dev libnghttp2-dev zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY link.sh /usr/local/bin/link.sh +RUN chmod +x /usr/local/bin/link.sh +ENV CN1_LINK_MODE=dynamic +ENTRYPOINT ["/usr/local/bin/link.sh"] diff --git a/vm/backend/docker/Containerfile.musl b/vm/backend/docker/Containerfile.musl new file mode 100644 index 00000000000..63e54eb5a2b --- /dev/null +++ b/vm/backend/docker/Containerfile.musl @@ -0,0 +1,42 @@ +# Builder image for fully static (musl) Linux backend binaries. +# +# The counterpart to Containerfile.glibc. The output depends on no libc at all, so +# it runs in a scratch or distroless image and the container's size is the +# binary's size. +# +# Alpine's prebuilt libcurl.a is compiled with brotli, libpsl, nghttp2 and c-ares, +# whose static archives are LTO objects lld cannot resolve. Rather than chase +# those, curl is built here with only what a server-side HTTP client needs: +# OpenSSL for TLS, zlib for content encoding, nothing else. Verification stays +# curl's, which is the reason for using curl instead of hand-rolled TLS. +# +# Only lib/ and include/ are built: the curl COMMAND needs perl to generate its +# man page (even with --disable-manual) and nothing here uses the command. +# +# Built once; every subsequent binary is just the clang invocation. +FROM alpine:3.20 + +ARG CURL_VER=8.7.1 + +RUN apk add --no-cache clang lld musl-dev openssl-dev openssl-libs-static \ + zlib-dev zlib-static nghttp2-dev nghttp2-static curl tar make + +RUN cd /tmp \ + && curl -fsSL "https://curl.se/download/curl-${CURL_VER}.tar.gz" -o curl.tar.gz \ + && tar xzf curl.tar.gz \ + && cd "curl-${CURL_VER}" \ + && ./configure --disable-shared --enable-static --with-openssl \ + --without-brotli --without-libpsl --without-nghttp2 --without-libidn2 \ + --without-zstd --disable-ares --disable-ldap --disable-ldaps \ + --disable-rtsp --disable-dict --disable-telnet --disable-tftp \ + --disable-pop3 --disable-imap --disable-smtp --disable-gopher \ + --disable-mqtt --disable-manual --prefix=/opt/curlstatic \ + && make -j"$(nproc)" -C lib \ + && make -C lib install \ + && make -C include install \ + && cd / && rm -rf /tmp/curl* + +COPY link.sh /usr/local/bin/link.sh +RUN chmod +x /usr/local/bin/link.sh +ENV CN1_LINK_MODE=static +ENTRYPOINT ["/usr/local/bin/link.sh"] diff --git a/vm/backend/docker/link.sh b/vm/backend/docker/link.sh new file mode 100644 index 00000000000..73d5b9e825e --- /dev/null +++ b/vm/backend/docker/link.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# Links the generated C in /src into /out/$CN1_OUT_NAME. +# +# The same script serves both builder images; what differs is CN1_LINK_MODE. +# +# static (musl/Alpine) a binary with no libc at all, which is what runs in a +# scratch or distroless image and what makes the +# container's size the binary's size. +# dynamic (glibc/Debian) linked against the distribution's libc and OpenSSL, for +# a base image that already carries them and patches them +# on its own schedule. +# +# -fwrapv -fno-strict-aliasing -fno-builtin-fmod(f) are MANDATORY for ParparVM's +# generated C (Java wrapping arithmetic; clang -O3 provably miscompiles without +# them). -static-pie is deliberately NOT used for the musl build: musl's static +# PIE and the crash handler's stack introspection disagree about the load base. +set -e +cd /src +OUT_NAME="${CN1_OUT_NAME:-bootstrap}" +COMMON="-O3 -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf" + +# CN1_LINK_DEBUG=1 keeps the symbol table and frame pointers so a debugger can +# name what it finds. Without it every backtrace from a deployed binary is a list +# of hex addresses, which is exactly as useful as no backtrace at all. +STRIP="-Wl,--strip-all" +if [ -n "${CN1_LINK_DEBUG:-}" ]; then + STRIP="" + COMMON="$COMMON -g -fno-omit-frame-pointer" +fi + +# The virtual-thread switch is assembly, so the .S files compile alongside the C. +# Globbing only *.c compiles the C half and fails at link with "undefined symbol: +# cn1VirtualThreadSwitch", which names the symbol but not the reason. +ASM_SOURCES="" +for f in *.S; do + [ -e "$f" ] && ASM_SOURCES="$ASM_SOURCES $f" +done + +if [ "${CN1_LINK_MODE:-static}" = "static" ]; then + # shellcheck disable=SC2086 + clang $COMMON -static -fuse-ld=lld -I. -I/opt/curlstatic/include \ + ${CN1_EXTRA_CFLAGS} *.c $ASM_SOURCES \ + -L/opt/curlstatic/lib -lcurl -lnghttp2 -lssl -lcrypto -lz -lm -lpthread \ + $STRIP \ + -o "/out/$OUT_NAME" +else + # shellcheck disable=SC2086 + clang $COMMON -I. ${CN1_EXTRA_CFLAGS} *.c $ASM_SOURCES \ + -lcurl -lnghttp2 -lssl -lcrypto -lz -lm -lpthread \ + $STRIP \ + -o "/out/$OUT_NAME" +fi + +echo "arch: $(uname -m) libc: ${CN1_LINK_MODE:-static}" +ls -l "/out/$OUT_NAME" +# Proof rather than intent: a "static" build that quietly picked up a shared libc +# would run here and fail in a scratch image, which is the worst place to find out. +if [ "${CN1_LINK_MODE:-static}" = "static" ]; then + if command -v ldd >/dev/null 2>&1 && ldd "/out/$OUT_NAME" 2>&1 | grep -q "=>"; then + echo "the static build has dynamic dependencies:" + ldd "/out/$OUT_NAME" + exit 1 + fi +fi diff --git a/vm/backend/generate-contract.sh b/vm/backend/generate-contract.sh new file mode 100755 index 00000000000..ec9f6da92b4 --- /dev/null +++ b/vm/backend/generate-contract.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# Generates the server half of the shared @RestClient contract. +# +# This runs the REAL goal a Codename One project runs -- cn1:process-annotations +# at PROCESS_CLASSES, from contract/pom.xml -- rather than a bespoke invocation of +# the processor. If this works, the production path works. +# +# The contract itself is compiled against the CN1 core (it names @GET, OnComplete +# and Response). The GENERATED classes reference nothing outside java.*, which is +# what lets them link into a server binary with no platform layer -- so the +# contract's own class is dropped afterwards and only the generated pair ships. +# +# With --if-needed it returns immediately when gen/ is already up to date, which +# is how build.sh and run-javase.sh can depend on it without paying for maven on +# every build. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +M2="${CN1_M2:-$REPO/.m2-repo}" + +IF_NEEDED=0 +if [ "$1" = "--if-needed" ]; then IF_NEEDED=1; fi + +up_to_date() { + [ -d gen ] || return 1 + [ -n "$(find gen -name '*.class' 2>/dev/null | head -1)" ] || return 1 + [ -z "$(find contract -name '*.java' -newer gen 2>/dev/null | head -1)" ] || return 1 + # The GENERATOR counts as an input too. Only the contract sources were + # checked, so editing RestServerAnnotationProcessor -- or just rebuilding the + # plugin -- left gen/ looking current, and build.sh and run-javase.sh went on + # exercising the previous dispatcher and codecs. A parity run then reported on + # a generator change that was not in the program it tested, which is the worst + # kind of green. + for artifact in \ + "$REPO/maven/codenameone-maven-plugin/target/classes" \ + "$REPO/maven/codenameone-maven-plugin/target"/codenameone-maven-plugin-*.jar + do + [ -e "$artifact" ] || continue + [ -z "$(find "$artifact" -newer gen 2>/dev/null | head -1)" ] || return 1 + done + return 0 +} + +if [ "$IF_NEEDED" = "1" ] && up_to_date; then + exit 0 +fi + +# One writer at a time. build.sh and run-javase.sh both call this, and the test +# suite forks several of them at once against this one working tree -- without a +# lock the second fork's javac reads gen/ while the first is between its `rm -rf` +# and its `cp`, and fails on classes that exist in both before and after. +# +# mkdir is the atomic primitive that exists everywhere; flock is not on macOS. +mkdir -p target +LOCK="$(pwd)/target/.contract.lock" +# A lock left behind by a killed process would block every later build forever, so +# one older than any plausible generation is taken as abandoned. +if [ -d "$LOCK" ] && [ -z "$(find "$LOCK" -maxdepth 0 -mmin -20 2>/dev/null)" ]; then + echo "removing an abandoned $LOCK" + rmdir "$LOCK" 2>/dev/null || true +fi +waited=0 +while ! mkdir "$LOCK" 2>/dev/null; do + waited=$((waited + 1)) + if [ "$waited" -gt 600 ]; then + echo "timed out waiting for $LOCK" + exit 1 + fi + sleep 1 +done +trap 'rmdir "$LOCK" 2>/dev/null || true' EXIT + +# Re-checked while holding the lock: the process we queued behind was very likely +# generating exactly what we were about to. +if [ "$IF_NEEDED" = "1" ] && up_to_date; then + exit 0 +fi + +# Checked here rather than at the top: --if-needed returns above without running +# maven, and the local Java SE loop should not demand a JDK 8 it never uses. +J8="${JDK_8_HOME:?set JDK_8_HOME to a JDK 8 home}" + +# The contract compiles against codenameone-core and is processed by the Codename +# One maven plugin, so both have to be in the local repo. Saying which ones are +# missing beats maven's "could not resolve" on an artifact nobody asked for +# directly -- this is the first thing a fresh checkout hits. +CN1_VERSION="$(sed -n 's/.*\(.*\)<\/cn1\.version>.*/\1/p' contract/pom.xml | head -1)" +for artifact in codenameone-core codenameone-maven-plugin; do + if [ ! -d "$M2/com/codenameone/$artifact/$CN1_VERSION" ]; then + echo "$artifact:$CN1_VERSION is not in $M2." + echo "Install it first:" + echo " (cd $REPO/maven && JAVA_HOME=\$JDK_8_HOME mvn -B -pl core,codenameone-maven-plugin \\" + echo " -am install -DskipTests -Plocal-dev-javase -Dmaven.repo.local=$M2)" + exit 1 + fi +done + +JAVA_HOME="$J8" mvn -q -B -f contract/pom.xml process-classes \ + -Dcn1.restServer=true -Dmaven.repo.local="$M2" + +rm -rf gen && mkdir -p gen +cp -r contract/target/classes/. gen/ +# The same goal generates BOTH halves. The client half -- Impl and +# cn1app.RestClientBootstrap -- belongs to the app: it calls +# com.codename1.io.rest.Rest, which needs a CodenameOneImplementation the server +# does not have, so shipping it would break the backend link. The contract +# interface goes for the same reason (it names OnComplete and Response). +# +# What stays: Server, Dispatcher, the DTOs and their Json codecs. +find gen -name '*Impl.class' -o -name '*Impl$*.class' | xargs -r rm -f +rm -rf gen/cn1app +# A contract type ships to the server exactly when a codec was generated for it: +# that is what makes it a DTO rather than the interface itself. Anything else from +# contract/ names OnComplete and Response and would not link. +for f in $(find contract -name '*.java'); do + rel="${f#contract/}" + cls="gen/${rel%.java}.class" + codec="gen/${rel%.java}Json.class" + if [ ! -f "$codec" ]; then + rm -f "$cls" "gen/${rel%.java}"'$'*.class 2>/dev/null || true + fi +done +echo "generated:"; find gen -name '*.class' | sort | sed 's/^/ /' diff --git a/vm/backend/impl/javase/com/codename1/backend/Crypto.java b/vm/backend/impl/javase/com/codename1/backend/Crypto.java new file mode 100644 index 00000000000..76ea6a92bc2 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Crypto.java @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.security.spec.KeySpec; +import java.util.ArrayList; +import java.util.List; + +import javax.crypto.Mac; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; + +/** + * Java SE twin of Crypto, on the JDK's own providers. + * + * Same rule as the translated one: nothing is implemented by hand. The + * constant-time compare is MessageDigest.isEqual, which the JDK documents as not + * short-circuiting; a loop written here would let the optimizer decide, and an + * early exit on the first differing byte lets a MAC be forged a byte at a time. + */ +public final class Crypto { + public static final int PASSWORD_ITERATIONS = 210000; + private static final int PASSWORD_SALT_BYTES = 16; + private static final int PASSWORD_HASH_BYTES = 32; + private static final SecureRandom RANDOM = new SecureRandom(); + + private Crypto() { + } + + public static byte[] sha256(byte[] data) { + if(data == null) { + return null; + } + try { + return MessageDigest.getInstance("SHA-256").digest(data); + } catch (Exception err) { + return null; + } + } + + /** + * PBKDF2-HMAC-SHA-256. Exposed because SCRAM-SHA-256 -- how PostgreSQL + * authenticates by default -- is defined in terms of it with the server's + * iteration count, which {@link #hashPassword} does not let a caller choose. + */ + public static byte[] pbkdf2Sha256(byte[] password, byte[] salt, int iterations, int length) + throws IOException { + return pbkdf2(password, salt, iterations, length); + } + + /** + * SHA-1, for the database wire protocols that specify it (MySQL's + * mysql_native_password). Never for anything this code chooses. + */ + public static byte[] sha1(byte[] data) { + return digest("SHA-1", data); + } + + /** MD5, for PostgreSQL's md5 authentication method. See {@link #sha1}. */ + public static byte[] md5(byte[] data) { + return digest("MD5", data); + } + + private static byte[] digest(String algorithm, byte[] data) { + if(data == null) { + return null; + } + try { + return java.security.MessageDigest.getInstance(algorithm).digest(data); + } catch (java.security.NoSuchAlgorithmException err) { + throw new IllegalStateException(algorithm + " is not available", err); + } + } + + public static byte[] hmacSha256(byte[] key, byte[] data) { + if(key == null || data == null) { + return null; + } + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(data); + } catch (Exception err) { + return null; + } + } + + public static byte[] randomBytes(int length) throws IOException { + if(length <= 0) { + throw new IOException("No secure randomness available"); + } + byte[] out = new byte[length]; + RANDOM.nextBytes(out); + return out; + } + + public static boolean equalsConstantTime(byte[] a, byte[] b) { + if(a == null || b == null) { + return false; + } + return MessageDigest.isEqual(a, b); + } + + public static String hashPassword(String password) throws IOException { + // A null password is a MISSING one, not an empty one. utf8(null) answers an + // empty array, so a handler that passed a DTO field the client never sent + // got a perfectly valid verifier -- and verifyPassword("", thatHash) then + // succeeds, which turns an omitted credential into an empty-password + // account. verifyPassword already refuses null; this is the other half. + if(password == null) { + throw new IllegalArgumentException("a password is required"); + } + byte[] salt = randomBytes(PASSWORD_SALT_BYTES); + byte[] hash = pbkdf2(utf8(password), salt, PASSWORD_ITERATIONS, PASSWORD_HASH_BYTES); + return "pbkdf2$" + PASSWORD_ITERATIONS + "$" + Base64Url.encode(salt) + + "$" + Base64Url.encode(hash); + } + + public static boolean verifyPassword(String password, String stored) { + if(password == null || stored == null) { + return false; + } + String[] parts = split(stored, '$'); + if(parts.length != 4 || !"pbkdf2".equals(parts[0])) { + return false; + } + int iterations; + try { + iterations = Integer.parseInt(parts[1]); + } catch (NumberFormatException err) { + return false; + } + byte[] salt = Base64Url.decode(parts[2]); + byte[] expected = Base64Url.decode(parts[3]); + if(salt == null || expected == null || iterations <= 0) { + return false; + } + // Non-EMPTY, not merely non-null. "pbkdf2$1$$" decodes to two empty arrays, + // pbkdf2 then derives zero bytes, and comparing an empty expectation with + // an empty derivation is TRUE -- so a stored row of that shape accepted + // every password. Base64Url.decode answers an empty array for an empty + // field, so the null check above never saw it. The floors are the standard + // minimums (RFC 8018 wants at least eight bytes of salt); anything this + // server writes is 16 and 32. + if(salt.length < 8 || expected.length < 16) { + return false; + } + try { + return equalsConstantTime(expected, pbkdf2(utf8(password), salt, iterations, expected.length)); + } catch (IOException err) { + return false; + } + } + + /** + * PBKDF2-HMAC-SHA256 over the password BYTES, per RFC 8018. + * + * Computed here rather than through PBEKeySpec, which takes chars and leaves the + * encoding to the provider: for PBKDF2WithHmacSHA256 that encoding is UTF-8, so + * a byte of 0xc3 handed over as a char came back out as TWO bytes. Mapping the + * UTF-8 bytes to chars first therefore did not preserve them -- it re-encoded + * them -- and the derived key stopped matching the native side, which passes the + * original octets to OpenSSL. The effect was confined to non-ASCII passwords: a + * hash written by one runtime that no longer verifies on the other, and a + * PostgreSQL SCRAM proof that simply does not authenticate. + */ + static byte[] pbkdf2(byte[] password, byte[] salt, int iterations, int length) + throws IOException { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + // SecretKeySpec rejects a zero-length key. HMAC pads the key to the block + // size with zeros, so a single zero byte and an empty key are the same + // key -- the substitution is exact rather than a workaround. + mac.init(new SecretKeySpec(password.length == 0 ? new byte[1] : password, + "HmacSHA256")); + int hLen = mac.getMacLength(); + byte[] out = new byte[length]; + byte[] counted = new byte[salt.length + 4]; + System.arraycopy(salt, 0, counted, 0, salt.length); + int done = 0; + for(int block = 1 ; done < length ; block++) { + counted[salt.length] = (byte)(block >>> 24); + counted[salt.length + 1] = (byte)(block >>> 16); + counted[salt.length + 2] = (byte)(block >>> 8); + counted[salt.length + 3] = (byte)block; + byte[] u = mac.doFinal(counted); + byte[] t = new byte[hLen]; + System.arraycopy(u, 0, t, 0, hLen); + for(int round = 1 ; round < iterations ; round++) { + u = mac.doFinal(u); + for(int iter = 0 ; iter < hLen ; iter++) { + t[iter] ^= u[iter]; + } + } + int take = length - done < hLen ? length - done : hLen; + System.arraycopy(t, 0, out, done, take); + done += take; + } + return out; + } catch (Exception err) { + throw new IOException("Key derivation failed"); + } + } + + static byte[] utf8(String value) { + try { + return value == null ? new byte[0] : value.getBytes("UTF-8"); + } catch (IOException err) { + return new byte[0]; + } + } + + private static String[] split(String value, char sep) { + List parts = new ArrayList(); + int pos = 0; + while(true) { + int next = value.indexOf(sep, pos); + if(next < 0) { + parts.add(value.substring(pos)); + break; + } + parts.add(value.substring(pos, next)); + pos = next + 1; + } + return parts.toArray(new String[parts.size()]); + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Db.java b/vm/backend/impl/javase/com/codename1/backend/Db.java new file mode 100644 index 00000000000..23fc1d36bd1 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Db.java @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Java SE twin of Db, over JDBC. + * + * The native runtime links SQLite directly; the JVM reaches the same file through + * the sqlite-jdbc driver. Row values are normalised to the SAME Java types the + * native side produces -- Long, Double, String, byte[], null -- so a handler that + * reads a column cannot behave differently between the two targets, which is the + * whole point of having a local dev loop at all. + * + * A path that already looks like a JDBC URL is passed through untouched, so the + * same code can point at MySQL or Postgres locally without a second API. + */ +public final class Db { + private Connection connection; + private long lastInsertId; + + private Db(Connection connection) { + this.connection = connection; + } + + public static Db open(String path) throws IOException { + // Db is the SQLite class on both arms: the packaged one hands this string + // straight to sqlite3_open. Accepting "jdbc:postgresql:..." here because a + // driver happens to be on the dev classpath let code work through + // cn1:backend and then fail once translated -- a dev loop that behaves + // differently from production, which is the one thing it must not do. + // Database is what speaks to those servers, on both arms. + if(path != null && path.startsWith("jdbc:") && !path.startsWith("jdbc:sqlite:")) { + throw new IOException("Db opens SQLite only, and the translated build " + + "would hand " + path + " to sqlite3_open. Use Database.open for " + + "PostgreSQL or MySQL."); + } + String url = path != null && path.startsWith("jdbc:") ? path : "jdbc:sqlite:" + path; + try { + Connection connection = DriverManager.getConnection(url); + connection.setAutoCommit(true); + return new Db(connection); + } catch (SQLException err) { + if(url.startsWith("jdbc:sqlite:")) { + // The usual cause is a dev classpath without the driver, and + // "No suitable driver" on its own does not say which one. + throw new IOException("Could not open " + url + " -- is sqlite-jdbc on " + + "the classpath? (" + err.getMessage() + ")"); + } + throw new IOException("Could not open " + url + ": " + err.getMessage()); + } + } + + public int execute(String sql, Object[] params) throws IOException { + Connection c = live(); + try { + if(params == null || params.length == 0) { + // PRAGMA and the transaction verbs are not all preparable on every + // driver, so a parameterless statement goes through Statement. + Statement statement = c.createStatement(); + try { + statement.execute(sql); + // Also here: an INSERT with literal values is still an INSERT, and + // the native implementation answers lastInsertRowid for it. Without + // this, the two backends disagree and the JavaSE one reports the id + // of some earlier parameterised insert. captureInsertId only assigns + // when the driver actually returns a key, so a PRAGMA or a CREATE + // leaves the previous value alone. + captureInsertId(statement); + int updated = statement.getUpdateCount(); + return updated < 0 ? 0 : updated; + } finally { + statement.close(); + } + } + PreparedStatement statement = c.prepareStatement(sql); + try { + bind(statement, params); + statement.execute(); + captureInsertId(statement); + int updated = statement.getUpdateCount(); + return updated < 0 ? 0 : updated; + } finally { + statement.close(); + } + } catch (SQLException err) { + throw new IOException("Statement failed: " + err.getMessage() + " [" + sql + "]"); + } + } + + public List query(String sql, Object[] params) throws IOException { + Connection c = live(); + try { + PreparedStatement statement = c.prepareStatement(sql); + try { + bind(statement, params); + ResultSet results = statement.executeQuery(); + try { + return readRows(results); + } finally { + results.close(); + } + } finally { + statement.close(); + } + } catch (SQLException err) { + throw new IOException("Query failed: " + err.getMessage() + " [" + sql + "]"); + } + } + + public Object transaction(Work body) throws Exception { + // BEGIN IMMEDIATE, not setAutoCommit(false), because that is what the + // PACKAGED arm does and the two must not disagree about concurrency. + // setAutoCommit(false) leaves the JDBC driver on SQLite's DEFERRED + // default, where a read-then-write transaction takes its read snapshot + // first and only asks for the write lock when it writes: two of them + // interleave, and the second fails SQLITE_BUSY on the upgrade instead + // of waiting at its start. So the same code that is well behaved here + // starts failing once it is packaged, which is the worst direction for + // a difference like this to run. IMMEDIATE takes the write lock up + // front, so the second transaction waits (bounded by busy_timeout). + execute("BEGIN IMMEDIATE", null); + boolean committed = false; + try { + Object result = body.run(this); + execute("COMMIT", null); + committed = true; + return result; + } finally { + if(!committed) { + try { + execute("ROLLBACK", null); + } catch (Exception err) { + // The original failure is the one worth reporting. + System.err.println("rollback failed: " + err); + } + } + } + } + + /** A unit of work run inside {@link #transaction}. */ + public interface Work { + Object run(Db db) throws Exception; + } + + public void enableWriteAheadLog() throws IOException { + query("PRAGMA journal_mode=WAL", null); + execute("PRAGMA synchronous=NORMAL", null); + } + + public void setBusyTimeout(int millis) throws IOException { + execute("PRAGMA busy_timeout=" + millis, null); + } + + public long lastInsertId() { + return lastInsertId; + } + + public void close() { + Connection c = connection; + connection = null; + if(c != null) { + try { + c.close(); + } catch (SQLException ignored) { + // already gone + } + } + } + + private Connection live() throws IOException { + Connection c = connection; + if(c == null) { + throw new IOException("Database is closed"); + } + return c; + } + + private void captureInsertId(Statement statement) { + try { + ResultSet keys = statement.getGeneratedKeys(); + if(keys != null) { + try { + if(keys.next()) { + lastInsertId = keys.getLong(1); + } + } finally { + keys.close(); + } + } + } catch (SQLException ignored) { + // Not every driver reports generated keys; the caller only misses an id. + } + } + + private static List readRows(ResultSet results) throws SQLException { + List rows = new ArrayList(); + ResultSetMetaData meta = results.getMetaData(); + int columns = meta.getColumnCount(); + String[] names = new String[columns]; + for(int iter = 0 ; iter < columns ; iter++) { + names[iter] = meta.getColumnLabel(iter + 1); + } + while(results.next()) { + Map row = new LinkedHashMap(); + for(int iter = 0 ; iter < columns ; iter++) { + row.put(names[iter], value(results, iter + 1)); + } + rows.add(row); + } + return rows; + } + + /** + * Normalises to the four types the native runtime hands back. Anything the + * driver gives us as some other class -- a java.sql.Timestamp, a BigDecimal -- + * becomes its string form, which is what SQLite's text affinity would have + * produced for the same column. + */ + private static Object value(ResultSet results, int index) throws SQLException { + Object raw = results.getObject(index); + if(raw == null || results.wasNull()) { + return null; + } + if(raw instanceof byte[]) { + return raw; + } + if(raw instanceof Number) { + if(raw instanceof Double || raw instanceof Float) { + return Double.valueOf(((Number)raw).doubleValue()); + } + if(raw instanceof Integer || raw instanceof Long + || raw instanceof Short || raw instanceof Byte) { + return Long.valueOf(((Number)raw).longValue()); + } + return Double.valueOf(((Number)raw).doubleValue()); + } + if(raw instanceof Boolean) { + return Long.valueOf(((Boolean)raw).booleanValue() ? 1 : 0); + } + return String.valueOf(raw); + } + + private static void bind(PreparedStatement statement, Object[] params) throws SQLException { + if(params == null) { + return; + } + for(int iter = 0 ; iter < params.length ; iter++) { + Object value = params[iter]; + int index = iter + 1; + if(value == null) { + statement.setNull(index, Types.NULL); + } else if(value instanceof String) { + statement.setString(index, (String)value); + } else if(value instanceof Integer || value instanceof Long + || value instanceof Short || value instanceof Byte) { + statement.setLong(index, ((Number)value).longValue()); + } else if(value instanceof Double || value instanceof Float) { + statement.setDouble(index, ((Number)value).doubleValue()); + } else if(value instanceof byte[]) { + statement.setBytes(index, (byte[])value); + } else if(value instanceof Boolean) { + statement.setLong(index, ((Boolean)value).booleanValue() ? 1 : 0); + } else { + statement.setString(index, String.valueOf(value)); + } + } + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Deadlines.java b/vm/backend/impl/javase/com/codename1/backend/Deadlines.java new file mode 100644 index 00000000000..e46ef587398 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Deadlines.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Read deadlines for the Java SE runtime. + * + * The translated target sets SO_RCVTIMEO on the descriptor and the kernel enforces + * it. An NIO channel has no such option, and a blocking channel read cannot be + * interrupted by a timer -- so a deadline is applied by putting the channel into + * non-blocking mode around a select() with a timeout. Without this a client that + * connects and says nothing holds a worker forever, and the pool is bounded. + */ +final class Deadlines { + private static final Map TIMEOUTS = new ConcurrentHashMap(); + + private Deadlines() { + } + + static void set(int fd, int millis) { + TIMEOUTS.put(Integer.valueOf(fd), Integer.valueOf(millis)); + } + + static void clear(int fd) { + TIMEOUTS.remove(Integer.valueOf(fd)); + } + + static int readWithDeadline(int fd, SocketChannel channel, ByteBuffer target) + throws IOException { + Integer timeout = TIMEOUTS.get(Integer.valueOf(fd)); + if(timeout == null || timeout.intValue() <= 0) { + return channel.read(target); + } + boolean wasBlocking = channel.isBlocking(); + Selector selector = null; + try { + channel.configureBlocking(false); + int n = channel.read(target); + if(n != 0) { + return n; + } + selector = Selector.open(); + channel.register(selector, SelectionKey.OP_READ); + if(selector.select(timeout.intValue()) == 0) { + throw new ServerSocket.TimeoutException("Read timed out on " + fd); + } + return channel.read(target); + } finally { + if(selector != null) { + selector.close(); + } + if(wasBlocking && channel.isOpen()) { + channel.configureBlocking(true); + } + } + } + + /** + * Writes the whole buffer, or gives up when the descriptor's deadline passes. + * + * A blocking write has no timeout of its own, so a client that requests a large + * response and then stops reading fills its receive window and parks the worker + * in write() for as long as it likes. Enough of them and every worker is held by + * a client that is doing nothing -- the native server has SO_SNDTIMEO for exactly + * this, and this runtime had nothing. + */ + static void writeWithDeadline(int fd, SocketChannel channel, ByteBuffer source) + throws IOException { + Integer timeout = TIMEOUTS.get(Integer.valueOf(fd)); + if(timeout == null || timeout.intValue() <= 0) { + while(source.hasRemaining()) { + if(channel.write(source) < 0) { + throw new IOException("Write failed on " + fd); + } + } + return; + } + boolean wasBlocking = channel.isBlocking(); + Selector selector = null; + try { + channel.configureBlocking(false); + while(source.hasRemaining()) { + int n = channel.write(source); + if(n < 0) { + throw new IOException("Write failed on " + fd); + } + if(n > 0) { + // Progress restarts the clock, so a slow but moving client is not + // cut off; only one that has stopped entirely is. + continue; + } + if(selector == null) { + selector = Selector.open(); + channel.register(selector, SelectionKey.OP_WRITE); + } + if(selector.select(timeout.intValue()) == 0) { + throw new ServerSocket.TimeoutException("Write timed out on " + fd); + } + selector.selectedKeys().clear(); + } + } finally { + if(selector != null) { + selector.close(); + } + if(wasBlocking && channel.isOpen()) { + channel.configureBlocking(true); + } + } + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Descriptors.java b/vm/backend/impl/javase/com/codename1/backend/Descriptors.java new file mode 100644 index 00000000000..c56e59d8494 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Descriptors.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.nio.channels.Channel; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Synthetic descriptors for the Java SE runtime. + * + * The shared code above -- HttpServer, StaticFiles -- deals in int descriptors, + * because on the translated target that is what they are. The JVM will not hand + * out a real fd portably, so this maps a synthetic int onto the channel it stands + * for. Nothing above needs to know. + * + * Ids start above the numbers a real process would use for stdin/stdout/stderr so + * a stray 0, 1 or 2 cannot be mistaken for a live descriptor. + */ +final class Descriptors { + private static final AtomicInteger NEXT = new AtomicInteger(64); + private static final Map ENTRIES = new ConcurrentHashMap(); + + private Descriptors() { + } + + static int add(Object entry) { + int id = NEXT.getAndIncrement(); + ENTRIES.put(Integer.valueOf(id), entry); + return id; + } + + static Object get(int id) { + return ENTRIES.get(Integer.valueOf(id)); + } + + static Object remove(int id) { + return ENTRIES.remove(Integer.valueOf(id)); + } + + static void closeQuietly(Object entry) { + if(entry instanceof Channel) { + try { + ((Channel)entry).close(); + } catch (IOException ignored) { + // already gone + } + } + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/FileIo.java b/vm/backend/impl/javase/com/codename1/backend/FileIo.java new file mode 100644 index 00000000000..2b7cf5a858c --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/FileIo.java @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.SocketChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; + +/** + * Java SE twin of FileIo. + * + * FileChannel.transferTo IS sendfile on Linux and macOS, so the zero-copy path is + * not lost here -- the JDK makes the same system call. The shared StaticFiles + * logic above is untouched. + */ +public final class FileIo { + private FileIo() { + } + + private static final class OpenFile { + final FileChannel channel; + final Path path; + /** + * Captured when the descriptor was opened, not read from the path later. + * + * A static asset replaced between the open and the stat would otherwise be + * described by its replacement while the bytes still came from the original + * channel: the response advertised the new length, timestamp and ETag and + * streamed the old file, which truncates or overruns whenever the two sizes + * differ. A descriptor is a snapshot, so its metadata has to be one too. + */ + final long size; + final long modified; + final boolean directory; + /** Whether the descriptor and the path agreed; see openRead. */ + final boolean consistent; + /** The file's identity as the path saw it, or null where unsupported. */ + final Object fileKey; + long position; + + OpenFile(FileChannel channel, Path path) { + this.channel = channel; + this.path = path; + long capturedSize = 0; + long capturedModified = 0; + boolean capturedDirectory = false; + boolean capturedConsistent = false; + Object capturedKey = null; + try { + if(channel != null) { + capturedSize = channel.size(); + } + BasicFileAttributes attributes = Files.readAttributes(path, + BasicFileAttributes.class); + capturedModified = attributes.lastModifiedTime().toMillis(); + capturedDirectory = attributes.isDirectory(); + capturedKey = attributes.fileKey(); + if(channel == null) { + capturedSize = attributes.size(); + capturedConsistent = true; + } else { + // The descriptor and the path describing the same file is what + // makes the size/mtime pair -- and so the ETag -- describe the + // bytes this descriptor will actually serve. + capturedConsistent = attributes.size() == capturedSize; + } + } catch (Exception ignored) { + // stat() reports the failure; there is nothing to do here. + } + this.size = capturedSize; + this.modified = capturedModified; + this.directory = capturedDirectory; + this.consistent = capturedConsistent; + this.fileKey = capturedKey; + } + } + + /** Unsupported on this runtime; see the ParparVM implementation. */ + public static final int BENEATH_UNSUPPORTED = -2; + + /** + * Always {@link #BENEATH_UNSUPPORTED} here. The local Java SE loop has no + * openat2, and a Java-side reimplementation would be the same racy + * open-then-check it replaces -- saying so lets the caller keep the older path + * rather than believe a check that did not happen. + */ + public static int openBeneath(String root, String relative) { + return BENEATH_UNSUPPORTED; + } + + public static int openRead(String path) { + try { + Path p = Paths.get(path); + if(Files.isDirectory(p)) { + // A directory has no channel, but the caller stats it and retries + // at the index file, so it must still get a descriptor back. + return Descriptors.add(new OpenFile(null, p)); + } + // Opened and stat'ed until the two AGREE. The size comes from the + // descriptor and the timestamp from the path, so a file replaced + // between them pairs the old bytes with the new file's mtime -- and + // StaticFiles builds its ETag from exactly that pair, so a client + // would cache the old content under the replacement's validator and + // be told 304 for as long as it asked. Java 8 has no fstat for a + // channel, so the race is detected rather than avoided: if the + // descriptor's size and the path's size disagree, the file changed + // under us and both are re-taken. A few attempts is plenty for an + // atomic replace; a file being rewritten continuously has no + // consistent validator to offer and gets the last pair read. + OpenFile opened = null; + for(int attempt = 0 ; attempt < 3 ; attempt++) { + // The file's IDENTITY across the open, because equal sizes prove + // nothing: a replacement by a file of the same length passes the + // size test, and then the old bytes are served under the new + // file's ETag -- so every later request for the new content is + // told 304 and the client caches the old representation for as + // long as it asks. An inode changes even when a length does not. + // Null where the filesystem has no such notion, and there the + // size test is all there is, which is what this did before. + Object keyBefore = fileKeyOf(p); + FileChannel channel = FileChannel.open(p, StandardOpenOption.READ); + OpenFile candidate = new OpenFile(channel, p); + boolean sameFile = keyBefore == null || candidate.fileKey == null + || keyBefore.equals(candidate.fileKey); + if((candidate.consistent && sameFile) || attempt == 2) { + opened = candidate; + break; + } + channel.close(); + } + return Descriptors.add(opened); + } catch (Exception err) { + return -1; + } + } + + /** A file's identity, or null when the filesystem does not report one. */ + private static Object fileKeyOf(Path p) { + try { + return Files.readAttributes(p, BasicFileAttributes.class).fileKey(); + } catch (Exception ignored) { + return null; + } + } + + public static int stat(int fd, long[] out) { + Object entry = Descriptors.get(fd); + if(!(entry instanceof OpenFile) || out == null || out.length < 3) { + return -1; + } + OpenFile file = (OpenFile)entry; + // From the descriptor, so the metadata and the bytes describe one file. + out[0] = file.size; + out[1] = file.modified; + out[2] = file.directory ? 1 : 0; + return 0; + } + + public static long sendFile(int socketFd, int fileFd, long offset, long count) { + Object file = Descriptors.get(fileFd); + Object socket = Descriptors.get(socketFd); + if(!(file instanceof OpenFile) || !(socket instanceof SocketChannel)) { + return -1; + } + FileChannel channel = ((OpenFile)file).channel; + if(channel == null) { + return -1; + } + try { + return channel.transferTo(offset, count, (WritableByteChannel)socket); + } catch (Exception err) { + return -1; + } + } + + public static boolean hasSendFile() { + // transferTo is sendfile underneath on every platform this runs on. + return true; + } + + public static int read(int fd, byte[] buffer, int offset, int length) { + Object entry = Descriptors.get(fd); + if(!(entry instanceof OpenFile) || buffer == null) { + return -1; + } + OpenFile file = (OpenFile)entry; + if(file.channel == null) { + return -1; + } + try { + ByteBuffer target = ByteBuffer.wrap(buffer, offset, length); + // A position is tracked explicitly so successive reads advance, which + // is what the shared code expects of a descriptor. + int n = file.channel.read(target, file.position); + if(n > 0) { + file.position += n; + } + return n < 0 ? 0 : n; + } catch (Exception err) { + return -1; + } + } + + public static String realPath(String path) { + try { + // Symlinks followed, as realpath does: the containment check above is + // only sound on a fully resolved path. + return Paths.get(path).toRealPath().toString(); + } catch (Exception err) { + return null; + } + } + + public static void close(int fd) { + Object entry = Descriptors.remove(fd); + if(entry instanceof OpenFile) { + FileChannel channel = ((OpenFile)entry).channel; + if(channel != null) { + try { + channel.close(); + } catch (IOException ignored) { + // already gone + } + } + } + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Http2.java b/vm/backend/impl/javase/com/codename1/backend/Http2.java new file mode 100644 index 00000000000..ca24066f8e8 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Http2.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Map; + +/** + * HTTP/2 is deliberately absent from the local Java SE runtime. + * + * h2 is only ever reached through ALPN on a TLS connection, and the local runtime + * does not terminate TLS (see Tls), so this could not be entered even if it were + * implemented. Keeping the shape and refusing at create() means the shared server + * code above needs no target-specific branch. + */ +public final class Http2 { + /** The ALPN protocol identifier, needed by the shared code even here. */ + public static final String ALPN = "h2"; + + private static final String UNSUPPORTED = + "HTTP/2 is not available in the local Java SE runtime -- it is reached " + + "through ALPN over TLS, which the local runtime does not terminate"; + + private Http2() { + } + + public static Http2 create() throws IOException { + throw new IOException(UNSUPPORTED); + } + + /** Mirrors the native runtime's stream shape so shared code compiles. */ + public static final class Stream { + final int id; + final String method; + final String path; + final String authority; + final Map headers; + final byte[] body; + + Stream(int id, String method, String path, String authority, Map headers, byte[] body) { + this.id = id; + this.method = method; + this.path = path; + this.authority = authority; + this.headers = headers; + this.body = body; + } + + public int getId() { + return id; + } + + public String getMethod() { + return method; + } + + public String getPath() { + return path; + } + + public String getAuthority() { + return authority; + } + + public Map getHeaders() { + return headers; + } + + /** The body as it ARRIVED, so the caller can check it before decoding. */ + public byte[] getBody() { + return body; + } + + public String getBodyAsString() { + if(body == null || body.length == 0) { + return null; + } + try { + return new String(body, "UTF-8"); + } catch (UnsupportedEncodingException err) { + return new String(body); + } + } + } + + public void receive(byte[] buffer, int offset, int length) throws IOException { + throw new IOException(UNSUPPORTED); + } + + public Stream nextRequest() { + return null; + } + + /** + * As {@link #respond}, with the body read from a descriptor rather than the heap. + * Unsupported here for the same reason the rest of this class is: the local run + * does not terminate TLS, so it never speaks HTTP/2. + */ + public boolean respondFile(int streamId, int status, String contentType, List extraHeaders, + int fd, long offset, long length) throws IOException { + throw new IOException(UNSUPPORTED); + } + + /** No session here, so there is no descriptor ceiling to enforce. */ + public static void setMaxFileBodies(int limit) { + } + + public boolean respond(int streamId, int status, String contentType, List extraHeaders, + byte[] body) throws IOException { + throw new IOException(UNSUPPORTED); + } + + /** Nothing is ever submitted here, so there is no ceiling to enforce. */ + public static void setMaxBodyBytes(long limit) { + } + + public byte[] drain() throws IOException { + throw new IOException(UNSUPPORTED); + } + + public boolean isAlive() { + return false; + } + + /** Nothing is ever submitted here, so nothing is ever outstanding. */ + public long pendingBodyBytes() { + return 0; + } + + /** Likewise: no session, so no file-backed body holds a descriptor. */ + public static int pendingBodyFiles() { + return 0; + } + + /** And nothing is submitted, so no body holds heap either. */ + public static long pendingBodyBytesAll() { + return 0; + } + + public void close() { + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Reactor.java b/vm/backend/impl/javase/com/codename1/backend/Reactor.java new file mode 100644 index 00000000000..fdd2b678daf --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Reactor.java @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.nio.channels.CancelledKeyException; +import java.nio.channels.SelectableChannel; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Java SE twin of the epoll/kqueue reactor, over a Selector. + * + * Two things about NIO that the native side gets for free and this has to work + * around, both of them thread-related: + * + * - register() from a thread other than the one inside select() blocks until the + * select returns. Workers hand descriptors back after finishing a request, so + * registrations are queued and applied by the selecting thread instead. + * - cancel() is lazy: the key survives until the next select, and a channel with a + * live key throws IllegalBlockingModeException when a worker flips it to + * blocking. remove() therefore flushes with selectNow(); it is only ever called + * from the selecting thread (the reactor loop) or for the listener during stop. + */ +public final class Reactor { + public static final int READ = 1; + public static final int WRITE = 2; + /** + * Deliver an event for this descriptor ONCE and then disarm it, until + * {@link #modify} re-arms it. + * + * This is what lets the worker threads poll the same set directly rather + * than a reactor thread dispatching to them: the kernel guarantees exactly + * one waiter is handed a given descriptor, so two workers cannot land on one + * connection. Without it a level-triggered set reports the same descriptor + * ready to every waiter at once. + */ + public static final int ONESHOT = 4; + + private final Selector selector; + private final Map keys = new ConcurrentHashMap(); + private final Deque pending = new ArrayDeque(); + + private Reactor(Selector selector) { + this.selector = selector; + } + + public static Reactor create() throws IOException { + return new Reactor(Selector.open()); + } + + /** Descriptors registered with {@link #ONESHOT}, so await() knows to disarm them. */ + private final java.util.Set oneshot = + java.util.Collections.synchronizedSet(new java.util.HashSet()); + + public void add(int fd, int events) throws IOException { + Integer key = Integer.valueOf(fd); + if((events & ONESHOT) != 0) { + oneshot.add(key); + } else { + oneshot.remove(key); + } + synchronized (pending) { + pending.add(new int[] {fd, events}); + } + selector.wakeup(); + } + + public void modify(int fd, int events) throws IOException { + SelectionKey key = keys.get(Integer.valueOf(fd)); + if(key == null) { + add(fd, events); + return; + } + try { + key.interestOps(toOps(events, key.channel())); + } catch (CancelledKeyException err) { + add(fd, events); + } + selector.wakeup(); + } + + public void remove(int fd) { + oneshot.remove(Integer.valueOf(fd)); + SelectionKey key = keys.remove(Integer.valueOf(fd)); + if(key == null) { + return; + } + key.cancel(); + try { + // Flush the cancellation now, so the worker about to take this + // descriptor can put it back into blocking mode. + selector.selectNow(); + } catch (IOException ignored) { + // A failed flush leaves the key for the next select to clear. + } + } + + public int await(int[] readyFds, int timeoutMillis) throws IOException { + applyPending(); + selector.select(timeoutMillis < 0 ? 0 : timeoutMillis); + int count = 0; + Iterator iterator = selector.selectedKeys().iterator(); + while(iterator.hasNext()) { + SelectionKey key = iterator.next(); + iterator.remove(); + if(count >= readyFds.length) { + break; + } + Object attachment = key.attachment(); + if(attachment instanceof Integer) { + // ONESHOT emulation: NIO has no equivalent, so clear the interest + // set the way epoll disarms a one-shot descriptor. modify() + // re-arms it. Without this the flag would be silently inert here + // and two threads polling one selector would both be handed the + // same connection -- the exact hazard ONESHOT exists to remove. + if(oneshot.contains(attachment)) { + key.interestOps(0); + } + readyFds[count++] = ((Integer)attachment).intValue(); + } + } + return count; + } + + public void close() { + try { + selector.close(); + } catch (IOException ignored) { + // already gone + } + keys.clear(); + } + + private void applyPending() { + while(true) { + int[] entry; + synchronized (pending) { + entry = pending.poll(); + } + if(entry == null) { + return; + } + Object channel = Descriptors.get(entry[0]); + if(!(channel instanceof SelectableChannel)) { + continue; + } + SelectableChannel selectable = (SelectableChannel)channel; + try { + SelectionKey key = selectable.register(selector, + toOps(entry[1], selectable), Integer.valueOf(entry[0])); + keys.put(Integer.valueOf(entry[0]), key); + } catch (Exception err) { + // A closed or already-cancelled channel simply does not come back. + keys.remove(Integer.valueOf(entry[0])); + } + } + } + + private static int toOps(int events, SelectableChannel channel) { + int ops = 0; + if((events & READ) != 0) { + // A server socket reports readiness to accept, not to read; the shared + // code above says READ for both, as poll does. + ops |= (channel.validOps() & SelectionKey.OP_ACCEPT) != 0 + ? SelectionKey.OP_ACCEPT : SelectionKey.OP_READ; + } + if((events & WRITE) != 0 && (channel.validOps() & SelectionKey.OP_WRITE) != 0) { + ops |= SelectionKey.OP_WRITE; + } + return ops; + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/ServerSocket.java b/vm/backend/impl/javase/com/codename1/backend/ServerSocket.java new file mode 100644 index 00000000000..8d934259382 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/ServerSocket.java @@ -0,0 +1,243 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.StandardSocketOptions; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; + +/** + * Java SE twin of ServerSocket, over NIO channels behind synthetic descriptors. + * + * Blocking mode is real here, as it is on the translated side: the reactor runs a + * Selector over non-blocking channels, and a worker that takes a connection flips + * it to blocking so request parsing is a read loop rather than a state machine. + */ +public final class ServerSocket { + private final ServerSocketChannel channel; + private final int fd; + + private ServerSocket(ServerSocketChannel channel, int fd) { + this.channel = channel; + this.fd = fd; + } + + /** Thrown when a read or write deadline expires. */ + public static final class TimeoutException extends IOException { + TimeoutException(String message) { + super(message); + } + } + + public static ServerSocket bind(String host, int port, int backlog) throws IOException { + ServerSocketChannel channel = ServerSocketChannel.open(); + try { + channel.setOption(StandardSocketOptions.SO_REUSEADDR, Boolean.TRUE); + channel.bind(host == null || "0.0.0.0".equals(host) + ? new InetSocketAddress(port) + : new InetSocketAddress(host, port), backlog); + return new ServerSocket(channel, Descriptors.add(channel)); + } catch (IOException err) { + channel.close(); + throw new IOException("Could not bind " + (host == null ? "*" : host) + ":" + port); + } + } + + public int getFd() { + return fd; + } + + public int getPort() { + try { + return ((InetSocketAddress)channel.getLocalAddress()).getPort(); + } catch (IOException err) { + return -1; + } + } + + public int accept() { + try { + SocketChannel client = channel.accept(); + if(client == null) { + return -1; + } + client.setOption(StandardSocketOptions.TCP_NODELAY, Boolean.TRUE); + return Descriptors.add(client); + } catch (IOException err) { + return -1; + } + } + + public void close() { + Descriptors.remove(fd); + try { + channel.close(); + } catch (IOException ignored) { + // already gone + } + } + + public static void setBlocking(int fd, boolean blocking) throws IOException { + Object entry = Descriptors.get(fd); + if(entry instanceof SocketChannel) { + ((SocketChannel)entry).configureBlocking(blocking); + return; + } + if(entry instanceof ServerSocketChannel) { + ((ServerSocketChannel)entry).configureBlocking(blocking); + return; + } + throw new IOException("Not a socket: " + fd); + } + + /** + * A read deadline. NIO channels have no SO_RCVTIMEO, so the deadline is + * enforced by the reader below; without one a silent client would hold a + * worker for as long as it liked, and the pool is bounded. + */ + public static void setTimeout(int fd, int millis) throws IOException { + Deadlines.set(fd, millis); + } + + /** + * Java SE twin of the readiness wait. See the translated version for why the + * shared code asks for this rather than juggling deadlines. + * + * A Selector is heavier than the single poll the translated side makes, which + * is acceptable here: this arm is the development loop, and its job is to + * behave the same, not to match the deployed binary's syscall count. + */ + /** + * A reusable per-thread read buffer of at least {@code capacity} bytes. + * + * The same array comes back on every call for a thread, so a server that reads + * through it allocates nothing per request. Its contents belong to the current + * callback only -- the next read on this thread overwrites them, so nothing may + * retain it or hand it to code that might. + * + * On the translated target the storage is a C buffer that the collector never + * allocated and never sweeps, so the read path contributes nothing at all to + * the allocation rate that paces the GC. Java SE cannot do that and returns an + * ordinary cached array; the observable contract is the same, which is the + * point -- only the allocation accounting differs. + */ + /** + * Read from {@code fd} into this thread's reusable buffer and return an array + * whose length is exactly the number of bytes read, or null at end of stream. + * + * On the translated target this allocates nothing and copies nothing: the array + * header and its storage are C memory the collector never touches, and the + * length is set per read so the caller can scan to {@code array.length}. Java SE + * cannot resize an array and returns a right-sized copy instead -- same + * contract, different allocation accounting. + * + * The bytes belong to the current callback on the current thread. Anything that + * must outlive either has to be copied out first. + */ + public static byte[] readIntoThreadBuffer(int fd, int capacity) throws IOException { + byte[] scratch = threadReadBuffer(capacity); + int n = read(fd, scratch, 0, capacity); + if(n <= 0) { + return null; + } + byte[] exact = new byte[n]; + System.arraycopy(scratch, 0, exact, 0, n); + return exact; + } + + public static byte[] threadReadBuffer(int capacity) { + byte[] cached = (byte[])THREAD_READ_BUFFER.get(); + if(cached == null || cached.length < capacity) { + cached = new byte[capacity]; + THREAD_READ_BUFFER.set(cached); + } + return cached; + } + + private static final ThreadLocal THREAD_READ_BUFFER = new ThreadLocal(); + + public static boolean awaitReadable(int fd, int timeoutMillis) throws IOException { + Object entry = Descriptors.get(fd); + if(!(entry instanceof SocketChannel)) { + throw new IOException("Not a socket: " + fd); + } + SocketChannel channel = (SocketChannel)entry; + boolean wasBlocking = channel.isBlocking(); + Selector selector = null; + try { + channel.configureBlocking(false); + selector = Selector.open(); + channel.register(selector, SelectionKey.OP_READ); + return selector.select(timeoutMillis) > 0; + } finally { + if(selector != null) { + selector.close(); + } + if(wasBlocking && channel.isOpen()) { + channel.configureBlocking(true); + } + } + } + + public static int read(int fd, byte[] buffer, int offset, int length) throws IOException { + Object entry = Descriptors.get(fd); + if(!(entry instanceof SocketChannel)) { + throw new IOException("Not a socket: " + fd); + } + SocketChannel channel = (SocketChannel)entry; + ByteBuffer target = ByteBuffer.wrap(buffer, offset, length); + if(channel.isBlocking()) { + // A blocking channel read cannot be interrupted by a timer, so the + // deadline is applied with a selector around it. + return Deadlines.readWithDeadline(fd, channel, target); + } + int n = channel.read(target); + return n; + } + + public static void write(int fd, byte[] buffer, int offset, int length) throws IOException { + Object entry = Descriptors.get(fd); + if(!(entry instanceof SocketChannel)) { + throw new IOException("Not a socket: " + fd); + } + SocketChannel channel = (SocketChannel)entry; + // Through the deadline, as reads are. A client that stops reading otherwise + // parks this worker in write() indefinitely. + Deadlines.writeWithDeadline(fd, channel, ByteBuffer.wrap(buffer, offset, length)); + } + + public static void closeFd(int fd) { + Object entry = Descriptors.remove(fd); + Deadlines.clear(fd); + Descriptors.closeQuietly(entry); + } + /** Cores available to this process. */ + public static int availableProcessors() { + return Runtime.getRuntime().availableProcessors(); + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Signals.java b/vm/backend/impl/javase/com/codename1/backend/Signals.java new file mode 100644 index 00000000000..9f14440e821 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Signals.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * Java SE twin of Signals. + * + * The JVM already turns SIGTERM and SIGINT into a shutdown hook, which runs on an + * ordinary thread and may do anything -- so the self-pipe the translated build + * needs has no counterpart here. SIGPIPE is likewise the JVM's problem: it sets + * the disposition itself, and a write to a departed peer surfaces as an + * IOException. + */ +public final class Signals { + private Signals() { + } + + public static boolean installShutdownHandler() { + return true; + } + + /** + * Blocks forever. There is nothing to wait for here -- the hook installed by + * onShutdown is what runs -- and returning would let a caller treat that as a + * signal having arrived. + */ + public static int awaitShutdownSignal() { + Object lock = new Object(); + synchronized(lock) { + while(true) { + try { + lock.wait(); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + return -1; + } + } + } + } + + /** + * Runs body from a JVM shutdown hook. + * + * The hook RETURNS when body is done, and body must not call System.exit: + * exiting from inside a shutdown hook blocks forever, because System.exit + * waits for the shutdown it is already part of. The JVM ends on its own once + * every hook has returned, so there is nothing left to do here. The ParparVM + * implementation of this method does have to end the process, which is why + * that belongs in these two files and not in any caller. + */ + public static void onShutdown(final Runnable body) { + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + public void run() { + System.out.println("shutdown requested"); + body.run(); + } + })); + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Tcp.java b/vm/backend/impl/javase/com/codename1/backend/Tcp.java new file mode 100644 index 00000000000..976d879fa41 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Tcp.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.Socket; + +import java.io.FileInputStream; +import java.security.KeyStore; +import java.security.cert.Certificate; +import java.security.cert.CertificateFactory; +import java.util.Collection; +import java.util.Iterator; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManagerFactory; + +/** + * Java SE twin of the translated Tcp. + * + * The public surface is identical on purpose. Everything above this -- Http, + * HttpServer, the Lambda runtime, the database and S3 clients -- is compiled from + * ONE shared source tree against whichever of the two implementations is on the + * path. Nothing above knows which target it is on, and there is no runtime lookup + * to get wrong. Divergence is caught by the runtime self-test, which runs against + * both. + */ +public final class Tcp { + private Socket socket; + private InputStream in; + private OutputStream out; + private boolean secure; + + private Tcp(Socket socket) throws IOException { + rebind(socket); + } + + private void rebind(Socket replacement) throws IOException { + this.socket = replacement; + this.in = replacement.getInputStream(); + this.out = replacement.getOutputStream(); + } + + public static Tcp connect(String host, int port, int timeoutMillis) throws IOException { + Socket s = new Socket(); + try { + s.connect(new InetSocketAddress(host, port), timeoutMillis); + // Nagle batches small writes, so on a request/response protocol a + // header would wait for its body. + s.setTcpNoDelay(true); + return new Tcp(s); + } catch (IOException err) { + try { + s.close(); + } catch (IOException ignored) { + // closing a socket that never connected + } + throw new IOException("Connection to " + host + ":" + port + " failed"); + } + } + + /** + * Upgrades this connection to TLS. See the translated twin for why this is an + * upgrade rather than a flag on connect. + * + * HTTPS endpoint identification is asked for explicitly: an SSLSocket made + * this way verifies the certificate chain by default but NOT that the name on + * it is the host we asked for, which is most of the protection. + */ + public void startTls(String host) throws IOException { + startTls(host, null); + } + + /** + * As {@link #startTls(String)}, verifying against the PEM bundle at `caFile` + * INSTEAD of the system trust store. See the translated twin for why. + */ + public void startTls(String host, String caFile) throws IOException { + if(secure) { + return; + } + SSLSocketFactory factory = caFile == null + ? (SSLSocketFactory)SSLSocketFactory.getDefault() + : factoryTrusting(caFile); + SSLSocket upgraded = (SSLSocket)factory + .createSocket(socket, host, socket.getPort(), true); + SSLParameters parameters = upgraded.getSSLParameters(); + parameters.setEndpointIdentificationAlgorithm("HTTPS"); + upgraded.setSSLParameters(parameters); + upgraded.startHandshake(); + rebind(upgraded); + secure = true; + } + + /** Whether this connection is encrypted. */ + public boolean isSecure() { + return secure; + } + + /** + * A factory that trusts exactly the certificates in one PEM bundle. The + * default trust store is deliberately NOT included: the caller named the + * roots it wants, and quietly adding more would defeat the point of naming + * them. + */ + private static SSLSocketFactory factoryTrusting(String caFile) throws IOException { + try { + KeyStore trust = KeyStore.getInstance(KeyStore.getDefaultType()); + trust.load(null, null); + CertificateFactory certificates = CertificateFactory.getInstance("X.509"); + FileInputStream in = new FileInputStream(caFile); + try { + Collection loaded = certificates.generateCertificates(in); + if(loaded.isEmpty()) { + throw new IOException("No certificates in " + caFile); + } + int index = 0; + Iterator it = loaded.iterator(); + while(it.hasNext()) { + trust.setCertificateEntry("ca" + (index++), it.next()); + } + } finally { + in.close(); + } + TrustManagerFactory managers = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm()); + managers.init(trust); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, managers.getTrustManagers(), null); + return context.getSocketFactory(); + } catch (IOException err) { + throw err; + } catch (Exception err) { + throw new IOException("Could not build a trust store from " + caFile + + ": " + err.getMessage()); + } + } + + public int read(byte[] buffer, int offset, int length) throws IOException { + return in.read(buffer, offset, length); + } + + public void write(byte[] buffer, int offset, int length) throws IOException { + out.write(buffer, offset, length); + out.flush(); + } + + public void close() { + try { + socket.close(); + } catch (IOException ignored) { + // already gone + } + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Tls.java b/vm/backend/impl/javase/com/codename1/backend/Tls.java new file mode 100644 index 00000000000..d8093b5eeb4 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Tls.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; + +/** + * TLS is deliberately absent from the local Java SE runtime. + * + * This twin exists so the shared server code compiles and runs unchanged on the + * JVM; it is the fast edit-run loop, not the deployment target. Terminating TLS + * here would mean a second, differently-behaving handshake and ALPN + * implementation (SSLEngine) whose bugs would not be the ones production has -- + * worse than not having it, because it would look like coverage. Run the native + * binary to exercise TLS; the integration suite does exactly that. + */ +public final class Tls { + private static final String UNSUPPORTED = + "TLS is not available in the local Java SE runtime -- run the native " + + "binary (or set CN1_BACKEND_TLS_CERT only there) to serve HTTPS"; + + private Tls() { + } + + public static Tls create(String certPath, String keyPath) throws IOException { + throw new IOException(UNSUPPORTED); + } + + public static Tls create(String certPath, String keyPath, boolean offerHttp2) + throws IOException { + throw new IOException(UNSUPPORTED); + } + + public long accept(int fd) { + throw new IllegalStateException(UNSUPPORTED); + } + + public void close() { + } + + static int read(long session, byte[] buffer, int offset, int length) throws IOException { + throw new IOException(UNSUPPORTED); + } + + static void write(long session, byte[] buffer, int offset, int length) throws IOException { + throw new IOException(UNSUPPORTED); + } + + static void closeSession(long session) { + } + + public static String negotiatedProtocol(long session) { + return null; + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/VirtualThread.java b/vm/backend/impl/javase/com/codename1/backend/VirtualThread.java new file mode 100644 index 00000000000..2997bc4c5ff --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/VirtualThread.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * The simulator has no virtual threads. + * + * They exist because ParparVM owns its whole translation and can switch a stack + * in a couple of nanoseconds; on a stock JVM the same idea is Loom's job, not + * ours. {@link #create} returning 0 is the documented "not available" answer and + * the server falls back to its pooled path, so behaviour here differs in + * scheduling only -- never in what a client sees. + */ +public final class VirtualThread { + private VirtualThread() { + } + + /** Always 0 here: not available, use the pool. */ + public static long create(int fd, int stackBytes) { + return 0; + } + + public static final int FINISHED = 0; + public static final int PARKED_IO = 1; + public static final int RUNNABLE = 2; + + public static int resume(long handle) { + return FINISHED; + } + + public static void free(long handle) { + } + + /** No virtual threads here. */ + public static int descriptorOf(long handle) { + return -1; + } + + public static boolean isVirtual() { + return false; + } + + /** No virtual threads here, so the server keeps the pool. */ + public static boolean supported() { + return false; + } + + /** No virtual threads here, so there is nothing to step aside for. */ + public static void yieldNow() { + } + + /** Nothing to report where there are none. */ + public static void report() { + } +} diff --git a/vm/backend/impl/javase/com/codename1/backend/Web.java b/vm/backend/impl/javase/com/codename1/backend/Web.java new file mode 100644 index 00000000000..1125f272636 --- /dev/null +++ b/vm/backend/impl/javase/com/codename1/backend/Web.java @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Java SE twin of Web, on HttpURLConnection. + * + * Certificate verification is the JVM's default trust store, and there is + * deliberately no way to turn it off here either -- an "insecure" flag is the kind + * of thing that ships enabled. + */ +public final class Web { + + /** + * ASCII lower case, because String.toLowerCase() is LOCALE SENSITIVE and this + * platform has no Locale to ask for the root one. On a device set to Turkish + * the I of an ASCII token folds to a dotless i, so a header stored under one + * spelling is looked up under another and getHeader answers null: nothing is + * thrown, nothing is logged, and the caller reads a header that is there as + * absent. A header name is ASCII by specification. Copied rather than shared; + * see CLAUDE.md. Both arms of Web carry it, because both index headers. + */ + private static String asciiLower(String value) { + if(value == null) { + return null; + } + StringBuilder out = new StringBuilder(value.length()); + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + out.append(c >= 'A' && c <= 'Z' ? (char)(c + 32) : c); + } + return out.toString(); + } + + private Web() { + } + + public static final class Result { + private final int status; + private final byte[] body; + private final String error; + private final Map headers; + + Result(int status, byte[] body, String error, Map headers) { + this.status = status; + this.body = body; + this.error = error; + this.headers = headers == null ? new LinkedHashMap() : headers; + } + + public int getStatus() { + return status; + } + + /** + * The response headers, lower-cased names to values. See the translated + * twin for why this exists. + */ + public Map getHeaders() { + return headers; + } + + /** One header by name, matched case-insensitively. Null when absent. */ + public String getHeader(String name) { + return name == null ? null : (String)headers.get(asciiLower(name)); + } + + public boolean isSuccess() { + return status >= 200 && status < 300; + } + + public byte[] getBody() { + return body; + } + + public String getBodyAsString() { + if(body == null) { + return null; + } + try { + return new String(body, "UTF-8"); + } catch (IOException err) { + return new String(body); + } + } + + public String getError() { + return error; + } + } + + public static Result get(String url) throws IOException { + return request("GET", url, null, null); + } + + public static Result getJson(String url, String bearerToken) throws IOException { + List headers = new ArrayList(); + headers.add("Accept: application/json"); + if(bearerToken != null) { + headers.add("Authorization: Bearer " + bearerToken); + } + return request("GET", url, headers, null); + } + + public static Result postJson(String url, String json, String bearerToken) throws IOException { + List headers = new ArrayList(); + headers.add("Content-Type: application/json"); + headers.add("Accept: application/json"); + if(bearerToken != null) { + headers.add("Authorization: Bearer " + bearerToken); + } + return request("POST", url, headers, json == null ? new byte[0] : json.getBytes("UTF-8")); + } + + public static Result request(String method, String url, List headers, byte[] body) + throws IOException { + if(url == null) { + throw new IOException("No URL"); + } + HttpURLConnection connection; + try { + connection = (HttpURLConnection)new URL(url).openConnection(); + } catch (IOException err) { + throw new IOException("Request to " + url + " failed: " + err.getMessage()); + } + try { + String verb = method == null ? "GET" : method; + try { + connection.setRequestMethod(verb); + } catch (java.net.ProtocolException unsupported) { + // HttpURLConnection has a FIXED set of verbs and PATCH is not in + // it, on every JDK this runs on. The packaged arm sends it through + // CURLOPT_CUSTOMREQUEST and does not care, so an integration that + // works once packaged fails here -- and the JDK's own message, + // "Invalid HTTP method: PATCH", says nothing about that being the + // difference. Reflecting over the private field is the usual trick + // and is not one: measured, it works on 8 and throws + // InaccessibleObjectException on 21 and 25, which are the versions + // this actually runs on. + throw new IOException("The local Java SE runtime cannot send " + verb + + " -- HttpURLConnection accepts a fixed set of verbs and this " + + "is not one of them. The packaged backend sends it normally, " + + "so this is a limitation of cn1:backend rather than of your " + + "code. Exercise this path against the packaged binary, or use " + + "POST with the override header your service expects."); + } + connection.setConnectTimeout(30000); + connection.setReadTimeout(30000); + // Following a redirect RESENDS the caller's headers to wherever it + // points, and HttpURLConnection carries every request property over + // -- it knows nothing about which of them is an X-Api-Key. A single + // 3xx from a service that has been taken over, or one that simply + // redirects off-domain, is then enough to hand the credential to the + // new host, with the caller never seeing where its header went. + // The packaged arm makes exactly this distinction (see the + // CURLOPT_FOLLOWLOCATION comment in cn1_backend_web.c) and the two + // must not disagree about it: whatever is unsafe there is unsafe + // here, and a difference between the arms is one more thing that + // only shows up after packaging. + connection.setInstanceFollowRedirects(headers == null || headers.isEmpty()); + connection.setRequestProperty("User-Agent", "codenameone-backend"); + if(headers != null) { + for(int iter = 0 ; iter < headers.size() ; iter++) { + String header = String.valueOf(headers.get(iter)); + int colon = header.indexOf(':'); + if(colon > 0) { + // addRequestProperty, not set: the packaged client appends + // every line it is given, so two Cookie or two extension + // lines both go out there while setRequestProperty kept + // only the last -- an integration that depends on a + // repeated header works once packaged and quietly sends + // half of what it meant to under cn1:backend. + connection.addRequestProperty(header.substring(0, colon).trim(), + header.substring(colon + 1).trim()); + } + } + } + if(body != null && body.length > 0) { + connection.setDoOutput(true); + connection.setFixedLengthStreamingMode(body.length); + OutputStream out = connection.getOutputStream(); + out.write(body); + out.flush(); + } + int status; + try { + status = connection.getResponseCode(); + } catch (IOException err) { + // No status at all: DNS, connect or TLS failed. The translated twin + // throws here too rather than reporting a status of -1. + throw new IOException("Request to " + url + " failed: " + err.getMessage()); + } + InputStream in = status >= 400 ? connection.getErrorStream() : connection.getInputStream(); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + if(in != null) { + byte[] chunk = new byte[8192]; + int n; + while((n = in.read(chunk)) > 0) { + buffer.write(chunk, 0, n); + } + } + // Lower-cased names, as the translated twin produces: a caller must + // not have to know which case this particular server chose. + Map responseHeaders = new LinkedHashMap(); + Map raw = connection.getHeaderFields(); + if(raw != null) { + java.util.Iterator it = raw.entrySet().iterator(); + while(it.hasNext()) { + Map.Entry entry = (Map.Entry)it.next(); + Object name = entry.getKey(); + if(name == null) { + continue; // the status line, which getHeaderFields keys as null + } + List values = (List)entry.getValue(); + if(values != null && !values.isEmpty()) { + responseHeaders.put(asciiLower(String.valueOf(name)), + String.valueOf(values.get(values.size() - 1))); + } + } + } + return new Result(status, buffer.toByteArray(), null, responseHeaders); + } finally { + connection.disconnect(); + } + } +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Crypto.java b/vm/backend/impl/parparvm/com/codename1/backend/Crypto.java new file mode 100644 index 00000000000..4f5085df2ec --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/Crypto.java @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; + +/** + * The crypto a server needs to authenticate a request. Every primitive comes from + * OpenSSL, which the backend already links for outbound TLS - none of it is + * implemented here, because hand-rolled HMAC and hand-rolled password hashing are + * the two most reliable ways to ship an authentication system that looks correct + * and is not. + */ +public final class Crypto { + /** + * PBKDF2 iterations for a stored password. Deliberately expensive: the cost is + * paid once per login and multiplied by every guess an attacker makes against a + * stolen table. + */ + public static final int PASSWORD_ITERATIONS = 210000; + private static final int PASSWORD_SALT_BYTES = 16; + private static final int PASSWORD_HASH_BYTES = 32; + + private Crypto() { + } + + public static byte[] sha256(byte[] data) { + return sha256Impl(data); + } + + /** + * SHA-1, for the database wire protocols that specify it (MySQL's + * mysql_native_password). Never for anything this code chooses: passwords go + * through {@link #hashPassword} and tokens through {@link #hmacSha256}. + */ + public static byte[] sha1(byte[] data) { + return sha1Impl(data); + } + + /** MD5, for PostgreSQL's md5 authentication method. See {@link #sha1}. */ + public static byte[] md5(byte[] data) { + return md5Impl(data); + } + + /** + * PBKDF2-HMAC-SHA-256. Exposed because SCRAM-SHA-256 -- how PostgreSQL + * authenticates by default -- is defined in terms of it with the server's + * iteration count, which {@link #hashPassword} does not let a caller choose. + */ + public static byte[] pbkdf2Sha256(byte[] password, byte[] salt, int iterations, int length) + throws IOException { + return pbkdf2(password, salt, iterations, length); + } + + public static byte[] hmacSha256(byte[] key, byte[] data) { + return hmacSha256Impl(key, data); + } + + /** Cryptographically secure bytes. Throws rather than returning weak ones. */ + public static byte[] randomBytes(int length) throws IOException { + byte[] out = randomBytesImpl(length); + if(out == null) { + throw new IOException("No secure randomness available"); + } + return out; + } + + /** + * Compares without leaking where two values first differ. An early exit on the + * first differing byte lets a MAC be forged one byte at a time. + */ + public static boolean equalsConstantTime(byte[] a, byte[] b) { + return equalsConstantTimeImpl(a, b); + } + + /** + * Hashes a password for storage. Returns "pbkdf2$iterations$salt$hash" with + * both binary parts base64url-encoded, so the iteration count travels with the + * hash and can be raised later without invalidating existing rows. + */ + public static String hashPassword(String password) throws IOException { + // A null password is a MISSING one, not an empty one. utf8(null) answers an + // empty array, so a handler that passed a DTO field the client never sent + // got a perfectly valid verifier -- and verifyPassword("", thatHash) then + // succeeds, which turns an omitted credential into an empty-password + // account. verifyPassword already refuses null; this is the other half. + if(password == null) { + throw new IllegalArgumentException("a password is required"); + } + byte[] salt = randomBytes(PASSWORD_SALT_BYTES); + byte[] hash = pbkdf2(utf8(password), salt, PASSWORD_ITERATIONS, PASSWORD_HASH_BYTES); + return "pbkdf2$" + PASSWORD_ITERATIONS + "$" + Base64Url.encode(salt) + "$" + Base64Url.encode(hash); + } + + /** False for any malformed stored value rather than throwing. */ + public static boolean verifyPassword(String password, String stored) { + if(password == null || stored == null) { + return false; + } + String[] parts = split(stored, '$'); + if(parts.length != 4 || !"pbkdf2".equals(parts[0])) { + return false; + } + int iterations; + try { + iterations = Integer.parseInt(parts[1]); + } catch (NumberFormatException err) { + return false; + } + byte[] salt = Base64Url.decode(parts[2]); + byte[] expected = Base64Url.decode(parts[3]); + if(salt == null || expected == null || iterations <= 0) { + return false; + } + // Non-EMPTY, not merely non-null. "pbkdf2$1$$" decodes to two empty arrays, + // pbkdf2 then derives zero bytes, and comparing an empty expectation with + // an empty derivation is TRUE -- so a stored row of that shape accepted + // every password. Base64Url.decode answers an empty array for an empty + // field, so the null check above never saw it. The floors are the standard + // minimums (RFC 8018 wants at least eight bytes of salt); anything this + // server writes is 16 and 32. + if(salt.length < 8 || expected.length < 16) { + return false; + } + byte[] actual = pbkdf2Impl(utf8(password), salt, iterations, expected.length); + return actual != null && equalsConstantTime(expected, actual); + } + + static byte[] pbkdf2(byte[] password, byte[] salt, int iterations, int length) throws IOException { + byte[] out = pbkdf2Impl(password, salt, iterations, length); + if(out == null) { + throw new IOException("Key derivation failed"); + } + return out; + } + + static byte[] utf8(String value) { + try { + return value == null ? new byte[0] : value.getBytes("UTF-8"); + } catch (IOException err) { + return new byte[0]; + } + } + + private static String[] split(String value, char sep) { + java.util.List parts = new java.util.ArrayList(); + int pos = 0; + while(true) { + int next = value.indexOf(sep, pos); + if(next < 0) { + parts.add(value.substring(pos)); + break; + } + parts.add(value.substring(pos, next)); + pos = next + 1; + } + String[] out = new String[parts.size()]; + for(int iter = 0 ; iter < out.length ; iter++) { + out[iter] = (String)parts.get(iter); + } + return out; + } + + private static native byte[] sha256Impl(byte[] data); + private static native byte[] sha1Impl(byte[] data); + private static native byte[] md5Impl(byte[] data); + private static native byte[] hmacSha256Impl(byte[] key, byte[] data); + private static native byte[] pbkdf2Impl(byte[] password, byte[] salt, int iterations, int length); + private static native byte[] randomBytesImpl(int length); + private static native boolean equalsConstantTimeImpl(byte[] a, byte[] b); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Db.java b/vm/backend/impl/parparvm/com/codename1/backend/Db.java new file mode 100644 index 00000000000..8e8f9ec8d40 --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/Db.java @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * SQLite persistence for server-side binaries, on the engine the translator + * already bundles. Not com.codename1.db.Database, which needs a + * CodenameOneImplementation for every call. + * + * Parameters are always bound, never interpolated: string concatenation into SQL + * is how injection happens, and a server parses input it did not write. + */ +public final class Db { + /** Column type codes, mirroring SQLITE_*. */ + private static final int TYPE_INTEGER = 1; + private static final int TYPE_FLOAT = 2; + private static final int TYPE_TEXT = 3; + private static final int TYPE_BLOB = 4; + private static final int TYPE_NULL = 5; + + private long handle; + + private Db(long handle) { + this.handle = handle; + } + + /** + * Opens (or creates) the database at the given path. ":memory:" gives a + * process-lifetime database, which is what a stateless function usually wants + * for a cache. + */ + public static Db open(String path) throws IOException { + long h = openImpl(path); + if(h == 0) { + throw new IOException("Could not open database at " + path); + } + return new Db(h); + } + + /** + * Runs a statement that returns no rows. Returns the number of rows changed. + */ + public int execute(String sql, Object[] params) throws IOException { + long stmt = prepare(sql, params); + try { + int rc = stepImpl(stmt); + if(rc < 0) { + throw new IOException("Statement failed: " + errorImpl(handle) + " [" + sql + "]"); + } + // Drain: a statement may return rows even when the caller ignores them. + while(rc == 1) { + rc = stepImpl(stmt); + if(rc < 0) { + throw new IOException("Statement failed: " + errorImpl(handle) + " [" + sql + "]"); + } + } + return changesImpl(handle); + } finally { + finalizeImpl(stmt); + } + } + + /** + * Runs a query and returns every row as a column-name to value map. Values are + * String, Long, Double or null, which is exactly what the JSON writer accepts. + */ + public List query(String sql, Object[] params) throws IOException { + long stmt = prepare(sql, params); + try { + List rows = new ArrayList(); + int columns = columnCountImpl(stmt); + String[] names = new String[columns]; + for(int iter = 0 ; iter < columns ; iter++) { + names[iter] = columnNameImpl(stmt, iter); + } + while(true) { + int rc = stepImpl(stmt); + if(rc < 0) { + throw new IOException("Query failed: " + errorImpl(handle) + " [" + sql + "]"); + } + if(rc == 0) { + return rows; + } + Map row = new LinkedHashMap(); + for(int iter = 0 ; iter < columns ; iter++) { + row.put(names[iter], columnValue(stmt, iter)); + } + rows.add(row); + } + } finally { + finalizeImpl(stmt); + } + } + + /** + * Runs body inside a transaction, committing when it returns and rolling back + * if it throws. A half-applied multi-statement change is the failure mode this + * exists to prevent, and getting the rollback right by hand at every call site + * is how it gets missed. + */ + public Object transaction(Work body) throws Exception { + execute("BEGIN IMMEDIATE", null); + boolean committed = false; + try { + Object result = body.run(this); + execute("COMMIT", null); + committed = true; + return result; + } finally { + if(!committed) { + try { + execute("ROLLBACK", null); + } catch (Exception err) { + // The original failure is the one worth reporting; a rollback + // that also fails must not replace it. + System.err.println("rollback failed: " + err); + } + } + } + } + + /** A unit of work run inside {@link #transaction}. */ + public interface Work { + Object run(Db db) throws Exception; + } + + /** + * Switches the database to write-ahead logging, which is what lets readers run + * while a writer is active. Worth doing once after open for anything that + * serves concurrent requests; pointless for :memory:. + */ + public void enableWriteAheadLog() throws IOException { + query("PRAGMA journal_mode=WAL", null); + execute("PRAGMA synchronous=NORMAL", null); + } + + /** + * How long a blocked writer waits for a competing one before giving up. Without + * this, two connections writing at once produce SQLITE_BUSY immediately rather + * than queueing. + */ + public void setBusyTimeout(int millis) throws IOException { + execute("PRAGMA busy_timeout=" + millis, null); + } + + /** The rowid the most recent insert produced. */ + public long lastInsertId() { + return lastInsertRowIdImpl(handle); + } + + public void close() { + if(handle != 0) { + long h = handle; + handle = 0; + closeImpl(h); + } + } + + private Object columnValue(long stmt, int index) { + switch(columnTypeImpl(stmt, index)) { + case TYPE_INTEGER: + return Long.valueOf(columnLongImpl(stmt, index)); + case TYPE_FLOAT: + return Double.valueOf(columnDoubleImpl(stmt, index)); + case TYPE_NULL: + return null; + case TYPE_BLOB: + return columnBlobImpl(stmt, index); + case TYPE_TEXT: + default: + return columnStringImpl(stmt, index); + } + } + + private long prepare(String sql, Object[] params) throws IOException { + if(handle == 0) { + throw new IOException("Database is closed"); + } + long stmt = prepareImpl(handle, sql); + if(stmt == 0) { + throw new IOException("Could not prepare: " + errorImpl(handle) + " [" + sql + "]"); + } + if(params != null) { + for(int iter = 0 ; iter < params.length ; iter++) { + bind(stmt, iter + 1, params[iter]); + } + } + return stmt; + } + + /** + * Binds one parameter, or fails. + * + * The status was discarded, so binding more parameters than the statement has + * placeholders -- SQLITE_RANGE -- was ignored and the statement executed anyway, + * with the unbound parameter reading as NULL. A mutation committed with the + * wrong values and nothing said so, where the JavaSE JDBC path throws. + */ + private static void bind(long stmt, int index, Object value) throws IOException { + int status; + if(value == null) { + status = bindNullImpl(stmt, index); + } else if(value instanceof String) { + status = bindStringImpl(stmt, index, (String)value); + } else if(value instanceof Integer || value instanceof Long + || value instanceof Short || value instanceof Byte) { + status = bindLongImpl(stmt, index, ((Number)value).longValue()); + } else if(value instanceof Double || value instanceof Float) { + status = bindDoubleImpl(stmt, index, ((Number)value).doubleValue()); + } else if(value instanceof byte[]) { + status = bindBlobImpl(stmt, index, (byte[])value); + } else if(value instanceof Boolean) { + status = bindLongImpl(stmt, index, ((Boolean)value).booleanValue() ? 1 : 0); + } else { + status = bindStringImpl(stmt, index, String.valueOf(value)); + } + if(status != 0) { // anything but SQLITE_OK + throw new IOException("Could not bind parameter " + index + + " (sqlite status " + status + "); check the parameter count"); + } + } + + private static native long openImpl(String path); + private static native int closeImpl(long handle); + private static native String errorImpl(long handle); + private static native long prepareImpl(long handle, String sql); + private static native int bindStringImpl(long stmt, int index, String value); + private static native int bindLongImpl(long stmt, int index, long value); + private static native int bindDoubleImpl(long stmt, int index, double value); + private static native int bindNullImpl(long stmt, int index); + private static native int bindBlobImpl(long stmt, int index, byte[] value); + private static native int stepImpl(long stmt); + private static native int columnCountImpl(long stmt); + private static native String columnNameImpl(long stmt, int index); + private static native int columnTypeImpl(long stmt, int index); + private static native String columnStringImpl(long stmt, int index); + private static native long columnLongImpl(long stmt, int index); + private static native double columnDoubleImpl(long stmt, int index); + private static native byte[] columnBlobImpl(long stmt, int index); + private static native void finalizeImpl(long stmt); + private static native int changesImpl(long handle); + private static native long lastInsertRowIdImpl(long handle); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/FileIo.java b/vm/backend/impl/parparvm/com/codename1/backend/FileIo.java new file mode 100644 index 00000000000..66d31f67686 --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/FileIo.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * The file-system calls a static handler needs, isolated so that everything else + * in [StaticFiles] -- ranges, conditional requests, MIME types, the containment + * check -- is pure Java and shared by every target. That logic is the part worth + * getting right once. + * + * There is one of these per target: the translated one goes to open/fstat/sendfile + * directly, the Java SE one to the JDK's channels. + */ +public final class FileIo { + private FileIo() { + } + + /** Opens for reading. The descriptor, or -1. */ + /** + * A descriptor for `relative` under `root`, or -1. Never a file outside `root`, + * and never by checking afterwards: the kernel refuses the escape while it + * resolves, so there is no window between the open and the check for a symlink + * to move through. + * + * Returns {@link #BENEATH_UNSUPPORTED} where the platform has no such call, so + * the caller can fall back rather than treat it as a missing file. + */ + public static int openBeneath(String root, String relative) { + return openBeneathImpl(root, relative); + } + + /** openBeneath cannot answer here; fall back to open plus a resolved-path check. */ + public static final int BENEATH_UNSUPPORTED = -2; + + public static int openRead(String path) { + return openReadImpl(path); + } + + /** + * Fills out[0]=size, out[1]=modified-time-millis, out[2]=1 for a directory. + * Taken from the OPEN DESCRIPTOR rather than the path: stat-then-open lets the + * file change in between, which is how a length header ends up disagreeing + * with the body. + */ + public static int stat(int fd, long[] out) { + return statImpl(fd, out); + } + + /** + * Sends bytes from a file straight to a socket, without them entering this + * process where the platform allows it. Returns how many moved, which may be + * fewer than asked; the caller loops. + */ + public static long sendFile(int socketFd, int fileFd, long offset, long count) { + return sendFileImpl(socketFd, fileFd, offset, count); + } + + /** True when [#sendFile] is a kernel copy rather than a read/write loop. */ + public static boolean hasSendFile() { + return hasSendFileImpl(); + } + + public static int read(int fd, byte[] buffer, int offset, int length) { + checkRange(buffer, offset, length); + return readImpl(fd, buffer, offset, length); + } + + /** + * The canonical path, symlinks followed. The static handler proves a resolved + * file is inside the document root with this: a check on the request string + * alone is defeated by an encoded traversal or by a symlink out of the tree. + */ + public static String realPath(String path) { + return realPathImpl(path); + } + + public static void close(int fd) { + closeImpl(fd); + } + + private static native int openBeneathImpl(String root, String relative); + + private static native int openReadImpl(String path); + private static native int statImpl(int fd, long[] out); + private static native long sendFileImpl(int socketFd, int fileFd, long offset, long count); + private static native boolean hasSendFileImpl(); + + /** + * Refuses a slice that does not lie inside the array. + * + * The natives below index the array through the pointer they are handed and + * ParparVM adds no bounds check of its own, so a bad offset is a native read + * or write of whatever is next in the heap rather than an exception. The + * JavaSE arm gets this free from its stream APIs, which is why such a bug is + * invisible on the simulator and only appears once packaged. The subtraction + * avoids the overflow that `offset + length` has. + */ + private static void checkRange(byte[] buffer, int offset, int length) { + if(buffer == null) { + throw new NullPointerException("buffer"); + } + if(offset < 0 || length < 0 || length > buffer.length - offset) { + throw new IndexOutOfBoundsException("offset " + offset + ", length " + + length + ", buffer " + buffer.length); + } + } + + private static native int readImpl(int fd, byte[] buffer, int offset, int length); + private static native String realPathImpl(String path); + private static native void closeImpl(int fd); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Http2.java b/vm/backend/impl/parparvm/com/codename1/backend/Http2.java new file mode 100644 index 00000000000..8a59c7f8bbe --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/Http2.java @@ -0,0 +1,361 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * One HTTP/2 connection, on nghttp2. + * + * The framing is not implemented here and should not be: HPACK alone is a static + * table, a dynamic table with eviction and Huffman coding, and flow control, + * stream state, CONTINUATION reassembly and GOAWAY are all their own problems. + * nghttp2 owns those. This class owns the shape of the boundary. + * + * Java PULLS from the session rather than being called back into. nghttp2 is + * callback-driven, but a C callback that reaches into the VM has to survive + * dead-code elimination and must not run while the collector is moving; the + * callbacks instead accumulate completed requests and this class takes them. + */ +public final class Http2 { + /** The ALPN identifier. There is no upgrade handshake for h2 over TLS. */ + public static final String ALPN = "h2"; + + private long session; + + private Http2(long session) { + this.session = session; + } + + /** A new server session, with the SETTINGS preface already queued. */ + public static Http2 create() throws IOException { + long s = createImpl(); + if(s == 0) { + throw new IOException("Could not create an HTTP/2 session"); + } + return new Http2(s); + } + + /** One request, once the client has finished sending it. */ + public static final class Stream { + final int id; + final String method; + final String path; + final String authority; + final Map headers; + final byte[] body; + + Stream(int id, String method, String path, String authority, Map headers, byte[] body) { + this.id = id; + this.method = method; + this.path = path; + this.authority = authority; + this.headers = headers; + this.body = body; + } + + public int getId() { + return id; + } + + public String getMethod() { + return method; + } + + /** Path and query, from the :path pseudo-header. */ + public String getPath() { + return path; + } + + /** From :authority, which is what Host is in HTTP/1.1. */ + public String getAuthority() { + return authority; + } + + /** Lower-cased names, as HTTP/2 requires them on the wire. */ + public Map getHeaders() { + return headers; + } + + /** The body as it ARRIVED, so the caller can check it before decoding. */ + public byte[] getBody() { + return body; + } + + public String getBodyAsString() { + if(body == null || body.length == 0) { + return null; + } + try { + return new String(body, "UTF-8"); + } catch (IOException err) { + return new String(body); + } + } + } + + /** Feeds received bytes to the session. */ + public void receive(byte[] buffer, int offset, int length) throws IOException { + checkRange(buffer, offset, length); + if(receiveImpl(session, buffer, offset, length) < 0) { + throw new IOException("HTTP/2 framing error"); + } + } + + /** + * The next completed request, or null. A stream is complete only when + * END_STREAM arrives -- on the HEADERS frame for a request with no body, on + * the last DATA frame otherwise. + */ + public Stream nextRequest() { + int id = nextRequestImpl(session); + if(id < 0) { + return null; + } + Map headers = new LinkedHashMap(); + int count = headerCountImpl(session); + for(int iter = 0 ; iter < count ; iter++) { + String name = headerNameImpl(session, iter); + if(name != null) { + String value = headerValueImpl(session, iter); + Object existing = headers.get(name); + if(existing == null) { + headers.put(name, value); + } else { + // A repeated field is COMBINED, not replaced. HTTP/2 lets a client + // split its cookies across several fields for better compression, + // and overwriting meant a session cookie sent in an earlier field + // vanished -- an authenticated request answered as anonymous. + // Cookie joins on "; " and everything else on ",", which is what + // RFC 9110 says a repeated field line means. + String separator = name.equalsIgnoreCase("cookie") ? "; " : ","; + headers.put(name, String.valueOf(existing) + separator + value); + } + } + } + return new Stream(id, methodImpl(session), pathImpl(session), + authorityImpl(session), headers, bodyImpl(session)); + } + + /** + * - `extraHeaders`: "name: value" strings. Connection-specific headers are + * dropped, because HTTP/2 forbids them, and names are lower-cased, because a + * capital letter is a protocol error the peer resets the stream over. + */ + public boolean respond(int streamId, int status, String contentType, List extraHeaders, byte[] body) + throws IOException { + int rc = respondImpl(session, streamId, String.valueOf(status), + headerLines(contentType, extraHeaders), body); + if(rc == OVER_BODY_BUDGET) { + // Not a failure: the body was refused because submitting it would + // cross the process-wide ceiling, and NOTHING was allocated or + // charged. The caller answers 503 instead. Reported rather than + // thrown because it is an ordinary load condition, and because the + // reservation has to be the same step as the allocation -- a limit + // the caller tests beforehand is two steps with a gap in the middle, + // which is how two sessions both passed a 64MB check and then held + // 80MB between them. + return false; + } + if(rc != 0) { + throw new IOException("Could not submit an HTTP/2 response on stream " + streamId); + } + return true; + } + + /** respondImpl's answer when the body would cross the ceiling. */ + static final int OVER_BODY_BUDGET = -2; + + /** + * The ceiling for outstanding response bodies across the process. + * + * Set once, and enforced natively where the memory is actually taken. + */ + public static void setMaxBodyBytes(long limit) { + setMaxBodyBytesImpl(limit); + } + + /** + * The ceiling for outstanding FILE-backed bodies across the process. + * + * Enforced natively for the same reason as the byte ceiling: the slot has to + * be taken in the same step as the descriptor, or every worker finishing at + * once passes the check before any of them counts. + */ + public static void setMaxFileBodies(int limit) { + setMaxFileBodiesImpl(limit); + } + + /** + * Responds with a range of an open file, without reading it into the heap. + * + * HTTP/2 cannot use sendfile -- the bytes have to become DATA frames -- but that + * is not a reason to materialise the whole file first. Reading it in cost the + * file's size in Java plus the same again in the native copy, so a large enough + * public file turned one request into an OutOfMemoryError, which the handler's + * `catch (Exception)` does not catch. The provider reads each frame straight out + * of the descriptor instead, so the memory is one frame regardless of size, and + * nghttp2's flow control decides the pace. + * + * The descriptor is owned by the session from here: it is closed when the stream + * reaches EOF, when it is reset early, and when the session is torn down. + */ + public boolean respondFile(int streamId, int status, String contentType, List extraHeaders, + int fd, long offset, long length) throws IOException { + int rc = respondFileImpl(session, streamId, String.valueOf(status), + headerLines(contentType, extraHeaders), fd, offset, length); + if(rc == OVER_BODY_BUDGET) { + // The descriptor ceiling was reached and NOTHING was taken -- the + // caller still owns the fd and has to close it. Reported rather than + // thrown because it is an ordinary load condition. + return false; + } + if(rc != 0) { + throw new IOException("Could not submit an HTTP/2 file response on stream " + + streamId); + } + return true; + } + + /** + * Runs the session's output side and returns the bytes to put on the wire. + * Empty when there is nothing pending. + */ + public byte[] drain() throws IOException { + if(pumpImpl(session) != 0) { + throw new IOException("HTTP/2 session failed"); + } + return drainImpl(session); + } + + /** False once the session is finished and the connection can be closed. */ + public boolean isAlive() { + return wantsMoreImpl(session); + } + + /** + * Heap held by response bodies that have been submitted and not yet fully + * written, which is NOT what drain() empties: that buffer is what nghttp2 + * has already serialised. nghttp2 pulls from a submitted body only as the + * peer's flow-control window allows, so a client that stops sending + * WINDOW_UPDATE leaves every body it asked for sitting here. A caller that + * keeps submitting has to look at this figure rather than at what it just + * handed over, because a flush that could write nothing frees nothing. + */ + public long pendingBodyBytes() { + return session == 0 ? 0 : pendingBodyBytesImpl(session); + } + + /** + * File-backed response bodies outstanding across the PROCESS, not this + * session. Such a body holds a descriptor and no heap, so it is invisible to + * pendingBodyBytes, and descriptors are a process resource: bounding them + * per connection still multiplies by the connection count, and exhausting + * them stops the process opening sockets or files at all. + */ + public static int pendingBodyFiles() { + return pendingBodyFilesImpl(); + } + + /** + * Response-body heap outstanding across the PROCESS rather than this session. + * The per-session figure says what one connection holds; a limit on that is a + * limit per connection, and the connection ceiling is in the thousands, so it + * bounds nothing about the machine. + */ + public static long pendingBodyBytesAll() { + return pendingBodyBytesAllImpl(); + } + + public void close() { + if(session != 0) { + long s = session; + session = 0; + destroyImpl(s); + } + } + + private static native long createImpl(); + + /** + * Refuses a slice that does not lie inside the array. + * + * The natives below index the array through the pointer they are handed and + * ParparVM adds no bounds check of its own, so a bad offset is a native read + * or write of whatever is next in the heap rather than an exception. The + * JavaSE arm gets this free from its stream APIs, which is why such a bug is + * invisible on the simulator and only appears once packaged. The subtraction + * avoids the overflow that `offset + length` has. + */ + private static void checkRange(byte[] buffer, int offset, int length) { + if(buffer == null) { + throw new NullPointerException("buffer"); + } + if(offset < 0 || length < 0 || length > buffer.length - offset) { + throw new IndexOutOfBoundsException("offset " + offset + ", length " + + length + ", buffer " + buffer.length); + } + } + + private static native int receiveImpl(long session, byte[] buffer, int offset, int length); + private static native int pumpImpl(long session); + private static native int pendingOutputImpl(long session); + private static native byte[] drainImpl(long session); + private static native int nextRequestImpl(long session); + private static native String methodImpl(long session); + private static native String pathImpl(long session); + private static native String authorityImpl(long session); + private static native int headerCountImpl(long session); + private static native String headerNameImpl(long session, int index); + private static native String headerValueImpl(long session, int index); + private static native byte[] bodyImpl(long session); + /** The header block both response forms send, as "name: value" lines. */ + private static String headerLines(String contentType, List extraHeaders) { + StringBuilder joined = new StringBuilder(); + joined.append("content-type: ").append(contentType == null + ? "application/octet-stream" : contentType); + if(extraHeaders != null) { + for(int iter = 0 ; iter < extraHeaders.size() ; iter++) { + joined.append('\n').append(String.valueOf(extraHeaders.get(iter))); + } + } + return joined.toString(); + } + + private static native int respondFileImpl(long session, int streamId, String status, + String headerLines, int fd, long offset, long length); + private static native void setMaxBodyBytesImpl(long limit); + + private static native void setMaxFileBodiesImpl(int limit); + + private static native int respondImpl(long session, int streamId, String status, + String headerLines, byte[] body); + private static native boolean wantsMoreImpl(long session); + private static native long pendingBodyBytesImpl(long session); + private static native int pendingBodyFilesImpl(); + private static native long pendingBodyBytesAllImpl(); + private static native void destroyImpl(long session); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Reactor.java b/vm/backend/impl/parparvm/com/codename1/backend/Reactor.java new file mode 100644 index 00000000000..30ffe282ffa --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/Reactor.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; + +/** + * Readiness notification over epoll (Linux) or kqueue (macOS/BSD). + * + * Level-triggered: the poller hands a ready descriptor to a worker and forgets + * about it until the worker gives it back. Edge-triggered would require draining + * every descriptor to EAGAIN on each wake-up, which is the opposite of that. + */ +public final class Reactor { + public static final int READ = 1; + public static final int WRITE = 2; + /** + * Deliver an event for this descriptor ONCE and then disarm it, until + * {@link #modify} re-arms it. + * + * This is what lets the worker threads poll the same set directly rather + * than a reactor thread dispatching to them: the kernel guarantees exactly + * one waiter is handed a given descriptor, so two workers cannot land on one + * connection. Without it a level-triggered set reports the same descriptor + * ready to every waiter at once. + */ + public static final int ONESHOT = 4; + + private int poller; + + private Reactor(int poller) { + this.poller = poller; + } + + public static Reactor create() throws IOException { + int p = createImpl(); + if(p < 0) { + throw new IOException("No readiness poller on this platform " + + "(epoll and kqueue are both unavailable)"); + } + return new Reactor(p); + } + + public void add(int fd, int events) throws IOException { + if(registerImpl(poller, fd, events, false) != 0) { + throw new IOException("Could not watch fd " + fd); + } + } + + public void modify(int fd, int events) throws IOException { + if(registerImpl(poller, fd, events, true) != 0) { + throw new IOException("Could not re-arm fd " + fd); + } + } + + public void remove(int fd) { + unregisterImpl(poller, fd); + } + + /** + * Blocks until something is ready, then fills readyFds and returns how many. + * A timeout below zero waits forever. + */ + public int await(int[] readyFds, int timeoutMillis) throws IOException { + int n = waitImpl(poller, readyFds, timeoutMillis); + if(n < 0) { + throw new IOException("Poller failed"); + } + return n; + } + + public void close() { + if(poller >= 0) { + int p = poller; + poller = -1; + ServerSocket.closeFd(p); + } + } + + private static native int createImpl(); + private static native int registerImpl(int poller, int fd, int events, boolean modify); + private static native int unregisterImpl(int poller, int fd); + private static native int waitImpl(int poller, int[] readyFds, int timeoutMillis); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/ServerSocket.java b/vm/backend/impl/parparvm/com/codename1/backend/ServerSocket.java new file mode 100644 index 00000000000..1bf42ee114a --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/ServerSocket.java @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; + +/** + * A listening TCP socket and the blocking read/write a worker uses once it owns a + * connection. Deliberately fd-based rather than object-per-socket: the reactor + * deals in descriptors and an extra object per idle connection is exactly the cost + * this design exists to avoid. + */ +public final class ServerSocket { + private int fd; + + private ServerSocket(int fd) { + this.fd = fd; + } + + /** + * - `host`: null or "0.0.0.0" to listen on every interface + * - `port`: 0 to let the OS choose, then ask {@link #getPort} + */ + public static ServerSocket bind(String host, int port, int backlog) throws IOException { + // The native side casts this to unsigned short, so 65536 became 0 and the + // process listened on an arbitrary port instead of refusing the setting. + // The Java SE arm rejects it through InetSocketAddress, so a PORT tested + // with cn1:backend behaved differently once packaged -- and an arbitrary + // port is the worst way to find out, because the process starts. + if(port < 0 || port > 65535) { + throw new IllegalArgumentException("port out of range: " + port); + } + int fd = bindImpl(host, port, backlog); + if(fd < 0) { + throw new IOException("Could not bind " + (host == null ? "*" : host) + ":" + port); + } + return new ServerSocket(fd); + } + + public int getFd() { + return fd; + } + + public int getPort() { + return boundPortImpl(fd); + } + + /** The accepted descriptor, or -1 when nothing was waiting. */ + public int accept() { + return acceptImpl(fd); + } + + public void close() { + if(fd >= 0) { + int f = fd; + fd = -1; + closeFdImpl(f); + } + } + + /** + * Blocking or non-blocking mode for one descriptor. The reactor needs + * non-blocking; a worker that owns a connection wants blocking, so it can read + * a request without a state machine. + */ + public static void setBlocking(int fd, boolean blocking) throws IOException { + if(setBlockingImpl(fd, blocking) != 0) { + throw new IOException("Could not change blocking mode on fd " + fd); + } + } + + /** Thrown when a read or write deadline expires. */ + public static final class TimeoutException extends IOException { + TimeoutException(String message) { + super(message); + } + } + + /** + * Applies a receive and send deadline. Without one a connection that opens and + * says nothing holds a worker forever, and the pool is bounded. + */ + public static void setTimeout(int fd, int millis) throws IOException { + if(setTimeoutImpl(fd, millis) != 0) { + throw new IOException("Could not set a deadline on fd " + fd); + } + } + + /** -1 at end of stream, as InputStream does. */ + /** + * Waits for the socket to become readable, for at most timeoutMillis. True if + * it is, false if the wait expired. + * + * One syscall, and it leaves the descriptor exactly as it was. The caller uses + * it between requests on a keep-alive connection, where the alternatives -- + * setting and restoring a receive deadline, or flipping to non-blocking and + * back -- cost two to four syscalls each way and disturb the deadline that + * governs a real request read. + */ + /** + * A reusable per-thread read buffer of at least {@code capacity} bytes. + * + * The same array comes back on every call for a thread, so a server that reads + * through it allocates nothing per request. Its contents belong to the current + * callback only -- the next read on this thread overwrites them, so nothing may + * retain it or hand it to code that might. + * + * On the translated target the storage is a C buffer that the collector never + * allocated and never sweeps, so the read path contributes nothing at all to + * the allocation rate that paces the GC. Java SE cannot do that and returns an + * ordinary cached array; the observable contract is the same, which is the + * point -- only the allocation accounting differs. + * + * Read from {@code fd} into this thread's reusable buffer and return an array + * whose length is exactly the number of bytes read, or null at end of stream. + * + * On the translated target this allocates nothing and copies nothing: the array + * header and its storage are C memory the collector never touches, and the + * length is set per read so the caller can scan to {@code array.length}. Java SE + * cannot resize an array and returns a right-sized copy instead -- same + * contract, different allocation accounting. + * + * The bytes belong to the current callback on the current thread. Anything that + * must outlive either has to be copied out first. + */ + public static byte[] readIntoThreadBuffer(int fd, int capacity) { + return readIntoThreadBufferImpl(fd, capacity); + } + + public static byte[] threadReadBuffer(int capacity) { + byte[] foreign = threadReadBufferImpl(capacity); + if(foreign != null) { + return foreign; + } + // The native refused (allocation failure). An ordinary array is correct, + // just not free, so the server keeps working rather than failing a request + // over an optimisation. + return new byte[capacity]; + } + + public static boolean awaitReadable(int fd, int timeoutMillis) throws IOException { + int rc = awaitReadableImpl(fd, timeoutMillis); + if(rc < 0) { + throw new IOException("Poll failed on fd " + fd); + } + return rc > 0; + } + + public static int read(int fd, byte[] buffer, int offset, int length) throws IOException { + checkRange(buffer, offset, length); + int n = readImpl(fd, buffer, offset, length); + if(n == -3) { + throw new TimeoutException("Read timed out on fd " + fd); + } + if(n < -1) { + throw new IOException("Read failed on fd " + fd); + } + return n; + } + + public static void write(int fd, byte[] buffer, int offset, int length) throws IOException { + checkRange(buffer, offset, length); + if(writeImpl(fd, buffer, offset, length) != length) { + throw new IOException("Write failed on fd " + fd); + } + } + + public static void closeFd(int fd) { + if(fd >= 0) { + closeFdImpl(fd); + } + } + + private static native int bindImpl(String host, int port, int backlog); + private static native int boundPortImpl(int fd); + private static native int acceptImpl(int serverFd); + private static native int setBlockingImpl(int fd, boolean blocking); + private static native int setTimeoutImpl(int fd, int millis); + /** + * A byte[] backed by this thread's C read buffer, handed over without a copy. + * + * The same object comes back on every call for a thread, its storage is never + * allocated by the collector, and it is not swept -- so the read path + * contributes nothing to the allocation rate that paces the GC. Returns null + * if the buffer cannot be provided, and the caller must then fall back to an + * ordinary array rather than assume it worked. + * + * The contents belong to the current callback ONLY. Nothing may retain this + * array or hand it to code that might: the next read on this thread overwrites + * it, and a grow moves the storage underneath it. + */ + static native byte[] threadReadBufferImpl(int capacity); + + static native byte[] readIntoThreadBufferImpl(int fd, int capacity); + + + private static native int awaitReadableImpl(int fd, int timeoutMillis); + + /** + * Refuses a slice that does not lie inside the array. + * + * The natives below index the array through the pointer they are handed and + * ParparVM adds no bounds check of its own, so a bad offset is a native read + * or write of whatever is next in the heap rather than an exception. The + * JavaSE arm gets this free from its stream APIs, which is why such a bug is + * invisible on the simulator and only appears once packaged. The subtraction + * avoids the overflow that `offset + length` has. + */ + private static void checkRange(byte[] buffer, int offset, int length) { + if(buffer == null) { + throw new NullPointerException("buffer"); + } + if(offset < 0 || length < 0 || length > buffer.length - offset) { + throw new IndexOutOfBoundsException("offset " + offset + ", length " + + length + ", buffer " + buffer.length); + } + } + + private static native int readImpl(int fd, byte[] buffer, int offset, int length); + private static native int writeImpl(int fd, byte[] buffer, int offset, int length); + private static native void closeFdImpl(int fd); + /** Cores available to this process. */ + public static int availableProcessors() { + int n = availableProcessorsImpl(); + return n > 0 ? n : 1; + } + + private static native int availableProcessorsImpl(); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Signals.java b/vm/backend/impl/parparvm/com/codename1/backend/Signals.java new file mode 100644 index 00000000000..f81caa2740d --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/Signals.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * Turns SIGTERM into an ordinary blocking call, so a server can shut down cleanly + * when its container asks it to. + * + * The handler itself does one async-signal-safe write() to a pipe, and this class + * turns that into an ordinary blocking read. Calling into the VM from a handler -- + * allocating, taking a monitor, touching the collector -- is undefined, and + * blocking the signals and calling sigwait() does not work either: ParparVM starts + * its collector thread before main(), so that thread never inherits the mask and + * dies on the default action. + */ +public final class Signals { + private Signals() { + } + + /** + * Installs the shutdown handlers and ignores SIGPIPE. Safe to call more than + * once. Writing to a socket whose peer has gone is routine for a server, and + * SIGPIPE's default action is to kill the process; ignored, the write returns + * an error like any other. + */ + public static boolean installShutdownHandler() { + return blockImpl() == 0; + } + + /** Blocks until SIGINT or SIGTERM arrives. Returns the signal number, or -1. */ + public static int awaitShutdownSignal() { + return awaitImpl(); + } + + /** + * Runs body on a dedicated thread when a shutdown signal arrives. + * blockShutdownSignals must already have been called. + */ + public static void onShutdown(final Runnable body) { + Thread t = new Thread(new Runnable() { + public void run() { + int signo = awaitShutdownSignal(); + if(signo > 0) { + System.out.println("signal " + signo + " received, shutting down"); + } + body.run(); + // Here rather than in body: stopping the server only unblocks the + // reactor, and every other thread is detached, so something has to + // end the process. Callers must NOT do this themselves -- the same + // body runs from a JVM shutdown hook under the JavaSE + // implementation, where exiting deadlocks. + System.exit(0); + } + }); + t.start(); + } + + private static native int blockImpl(); + private static native int awaitImpl(); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Tcp.java b/vm/backend/impl/parparvm/com/codename1/backend/Tcp.java new file mode 100644 index 00000000000..0fec25c325e --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/Tcp.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; + +/** + * Blocking TCP client socket for server-side (clean-target) binaries. Deliberately + * not com.codename1.io.Socket: that routes through CodenameOneImplementation, which + * a translated server binary does not have. + */ +public final class Tcp { + private long handle; + /** An OpenSSL session once startTls has run; 0 while the socket is plaintext. */ + private long tls; + + private Tcp(long handle) { + this.handle = handle; + } + + public static Tcp connect(String host, int port, int timeoutMillis) throws IOException { + // The same range ServerSocket.bind refuses, and for the same reason: the + // native side renders this into the service string getaddrinfo parses, and + // a value past 65535 does not fail there -- glibc wraps it, so 65536 dials + // port 0 and 99999 dials 34463. The Java SE arm rejects it outright, so a + // malformed database URL reached a DIFFERENT port only once packaged. + if(port < 0 || port > 65535) { + throw new IllegalArgumentException("port out of range: " + port); + } + // And the timeout, for the same reason one line up: the native side reads + // every NON-POSITIVE value as "block with no deadline", so a negative one + // hangs a packaged server for the OS TCP timeout while the Java SE arm + // fails immediately out of Socket.connect. Zero keeps its documented + // meaning; below zero is not a shorter wait, it is a different API. + if(timeoutMillis < 0) { + throw new IllegalArgumentException("connect timeout must not be negative: " + + timeoutMillis); + } + long h = connectImpl(host, port, timeoutMillis); + if(h == 0) { + throw new IOException("Connection to " + host + ":" + port + " failed"); + } + return new Tcp(h); + } + + /** + * Upgrades this connection to TLS, verifying the peer certificate against the + * system trust store and against `host`. + * + * An upgrade rather than a secure connect because that is the shape the + * database protocols need: PostgreSQL and MySQL both begin in plaintext and + * ask to start TLS mid-conversation, so a connect-time flag could not express + * it. Calling it immediately after connect gives the ordinary secure-connect + * behaviour. + */ + public void startTls(String host) throws IOException { + startTls(host, null); + } + + /** + * As {@link #startTls(String)}, verifying against the PEM bundle at `caFile` + * INSTEAD of the system trust store. + * + * This is what a managed database needs: RDS, Cloud SQL and the like present + * certificates from a private CA, and a development container presents one it + * generated for itself. Falling back to the system store when the named bundle + * fails to load would verify against roots the caller deliberately did not + * choose, so that is an error rather than a fallback. + */ + public void startTls(String host, String caFile) throws IOException { + checkOpen(); + if(tls != 0) { + return; + } + long session = startTlsImpl(handle, host, caFile); + if(session == 0) { + throw new IOException("TLS handshake with " + host + " failed: " + tlsErrorImpl()); + } + tls = session; + } + + /** + * Refuses a slice that does not lie inside the array. + * + * recv() and SSL_read() index the array straight through the pointer they are + * given, and ParparVM adds no bounds check of its own, so a bad offset here is + * a native read or write of whatever is next in the heap rather than an + * exception. The JavaSE implementation gets this free from the stream API, + * which is why the same code is safe on the simulator and unsafe only once it + * is packaged. The subtraction avoids the overflow `offset + length` has. + */ + private static void checkRange(byte[] buffer, int offset, int length) { + if(buffer == null) { + throw new NullPointerException("buffer"); + } + if(offset < 0 || length < 0 || length > buffer.length - offset) { + throw new IndexOutOfBoundsException("offset " + offset + ", length " + + length + ", buffer " + buffer.length); + } + } + + /** Whether this connection is encrypted. */ + public boolean isSecure() { + return tls != 0; + } + + /** + * Reads up to length bytes. Returns -1 at end of stream, matching InputStream. + */ + public int read(byte[] buffer, int offset, int length) throws IOException { + checkOpen(); + checkRange(buffer, offset, length); + int n = tls == 0 ? readImpl(handle, buffer, offset, length) + : tlsReadImpl(tls, buffer, offset, length); + if(n < -1) { + throw new IOException("Socket read failed"); + } + return n; + } + + public void write(byte[] buffer, int offset, int length) throws IOException { + checkOpen(); + checkRange(buffer, offset, length); + int n = tls == 0 ? writeImpl(handle, buffer, offset, length) + : tlsWriteImpl(tls, buffer, offset, length); + if(n != length) { + throw new IOException("Socket write failed"); + } + } + + public void close() { + if(tls != 0) { + long t = tls; + tls = 0; + tlsCloseImpl(t); + } + if(handle != 0) { + long h = handle; + handle = 0; + closeImpl(h); + } + } + + private void checkOpen() throws IOException { + if(handle == 0) { + throw new IOException("Socket closed"); + } + } + + private static native long connectImpl(String host, int port, int timeoutMillis); + private static native int readImpl(long handle, byte[] buffer, int offset, int length); + private static native int writeImpl(long handle, byte[] buffer, int offset, int length); + private static native int closeImpl(long handle); + private static native long startTlsImpl(long handle, String host, String caFile); + private static native String tlsErrorImpl(); + private static native int tlsReadImpl(long session, byte[] buffer, int offset, int length); + private static native int tlsWriteImpl(long session, byte[] buffer, int offset, int length); + private static native void tlsCloseImpl(long session); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Tls.java b/vm/backend/impl/parparvm/com/codename1/backend/Tls.java new file mode 100644 index 00000000000..1ebad51233f --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/Tls.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; + +/** + * Server-side TLS. One context for the process, one session per connection. + * + * The handshake runs on the worker that picks a connection up, not on the reactor + * thread: a handshake is several round trips, and doing it on the reactor would + * block every other connection behind one slow client. + * + * A TLS connection costs an SSL object, so the "an idle connection allocates + * nothing" property of the plain server does not hold here -- that is inherent to + * TLS, not a choice. It is also why static files lose their zero-copy path under + * TLS: sendfile works because the kernel moves bytes it never looks at, and + * encrypted bytes have to be produced in user space. + */ +public final class Tls { + private long context; + + private Tls(long context) { + this.context = context; + } + + /** + * - `certPath`: PEM certificate chain, leaf first + * - `keyPath`: PEM private key + */ + public static Tls create(String certPath, String keyPath) throws IOException { + return create(certPath, keyPath, false); + } + + /** + * - `offerHttp2`: advertise "h2" in ALPN. There is no upgrade handshake for + * HTTP/2 over TLS, so a server that does not advertise it here will never + * speak it however complete the rest of its implementation is. + */ + public static Tls create(String certPath, String keyPath, boolean offerHttp2) throws IOException { + long ctx = createContextImpl(certPath, keyPath, offerHttp2); + if(ctx == 0) { + throw new IOException("Could not load the certificate and key from " + + certPath + " and " + keyPath); + } + return new Tls(ctx); + } + + /** + * Runs the handshake on an already-blocking descriptor. Returns 0 when it + * fails, which is ordinary traffic -- a scanner, a client with no common + * cipher, or a plaintext request sent to the TLS port. + */ + public long accept(int fd) { + return acceptImpl(context, fd); + } + + public void close() { + if(context != 0) { + long c = context; + context = 0; + freeContextImpl(c); + } + } + + /** -1 at end of stream, as InputStream does. */ + static int read(long session, byte[] buffer, int offset, int length) throws IOException { + checkRange(buffer, offset, length); + int n = readImpl(session, buffer, offset, length); + if(n < -1) { + throw new IOException("TLS read failed"); + } + return n; + } + + static void write(long session, byte[] buffer, int offset, int length) throws IOException { + checkRange(buffer, offset, length); + if(writeImpl(session, buffer, offset, length) != length) { + throw new IOException("TLS write failed"); + } + } + + static void closeSession(long session) { + if(session != 0) { + closeImpl(session); + } + } + + /** The protocol ALPN settled on: "h2", "http/1.1", or null. */ + public static String negotiatedProtocol(long session) { + return negotiatedProtocolImpl(session); + } + + private static native long createContextImpl(String certPath, String keyPath, boolean offerHttp2); + private static native String negotiatedProtocolImpl(long session); + private static native void freeContextImpl(long handle); + private static native long acceptImpl(long context, int fd); + + /** + * Refuses a slice that does not lie inside the array. + * + * The natives below index the array through the pointer they are handed and + * ParparVM adds no bounds check of its own, so a bad offset is a native read + * or write of whatever is next in the heap rather than an exception. The + * JavaSE arm gets this free from its stream APIs, which is why such a bug is + * invisible on the simulator and only appears once packaged. The subtraction + * avoids the overflow that `offset + length` has. + */ + private static void checkRange(byte[] buffer, int offset, int length) { + if(buffer == null) { + throw new NullPointerException("buffer"); + } + if(offset < 0 || length < 0 || length > buffer.length - offset) { + throw new IndexOutOfBoundsException("offset " + offset + ", length " + + length + ", buffer " + buffer.length); + } + } + + private static native int readImpl(long session, byte[] buffer, int offset, int length); + private static native int writeImpl(long session, byte[] buffer, int offset, int length); + private static native void closeImpl(long session); +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/VirtualThread.java b/vm/backend/impl/parparvm/com/codename1/backend/VirtualThread.java new file mode 100644 index 00000000000..fbd56bf5e68 --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/VirtualThread.java @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * A thread of control that is not an OS thread. + * + * One of these per connection is what lets a server keep a context per client + * without keeping an OS THREAD per client. The difference is not stylistic: a + * handoff between OS threads measured 21181ns on the machine this was built on, + * and switching a virtual thread measured 2.6ns. + * + * A virtual thread runs until it finishes or until it asks for bytes that have + * not arrived, at which point it parks and the host thread goes and runs another + * one. Parking happens inside the ordinary blocking calls, so the code a virtual + * thread runs is written in the plain blocking style and does not know it is not + * a thread -- which is the reason to have them rather than callbacks. + */ +public final class VirtualThread { + private VirtualThread() { + } + + /** + * A virtual thread that will serve `fd` when first resumed. + * + * The stack is the C stack only. Java locals and the operand stack live in + * the virtual thread's own VM state, which is mapped lazily, so what this + * size buys is call DEPTH rather than data: it holds the C activation + * records of the Java methods the connection is nested inside. + * + * @return a handle, or 0 if the stack could not be allocated + */ + public static long create(int fd, int stackBytes) { + return createImpl(fd, stackBytes); + } + + /** {@link #resume}: the connection is done and the handle should be freed. */ + public static final int FINISHED = 0; + /** {@link #resume}: waiting for bytes; its descriptor goes back to the poller. */ + public static final int PARKED_IO = 1; + /** + * {@link #resume}: it gave up its turn but is ready to run again NOW. + * + * It is waiting on something that is not its socket -- the collector's + * allocation backpressure, or its own fairness yield. Putting it on the + * poller instead would wait for a client that is waiting for the response + * this virtual thread owes it, and the connection would hang for ever. + */ + public static final int RUNNABLE = 2; + + /** Run it until it parks, yields or finishes. One of the three constants. */ + public static int resume(long handle) { + return resumeImpl(handle); + } + + /** The descriptor this virtual thread serves, or -1. */ + public static int descriptorOf(long handle) { + return descriptorImpl(handle); + } + + /** Release it. Only valid once {@link #resume} has returned FINISHED. */ + public static void free(long handle) { + freeImpl(handle); + } + + /** + * Step aside so the host thread can run another virtual thread, without + * waiting for anything. + * + * Parking happens by itself when bytes have not arrived. This is for the + * other case: a virtual thread that COULD keep going but has had its turn. + * A no-op when the caller is not a virtual thread. + */ + public static void yieldNow() { + yieldImpl(); + } + + /** Whether the caller is running on a virtual thread rather than a host thread. */ + public static boolean isVirtual() { + return isVirtualImpl(); + } + + /** + * Whether this build has virtual threads at all, which is not the same + * question as isVirtual(). The context switch is compiled in only on + * non-Windows aarch64/x86_64; elsewhere create() can only ever return 0. + * The server asks this to pick its default poll mode. + */ + public static boolean supported() { + return supportedImpl(); + } + + private static native long createImpl(int fd, int stackBytes); + private static native int resumeImpl(long handle); + private static native int descriptorImpl(long handle); + private static native void freeImpl(long handle); + private static native boolean isVirtualImpl(); + private static native boolean supportedImpl(); + private static native void yieldImpl(); + private static native void reportImpl(); + + /** Print created/finished/freed counts to stderr, for diagnosis. */ + public static void report() { + reportImpl(); + } +} diff --git a/vm/backend/impl/parparvm/com/codename1/backend/Web.java b/vm/backend/impl/parparvm/com/codename1/backend/Web.java new file mode 100644 index 00000000000..fa2cadb7f5e --- /dev/null +++ b/vm/backend/impl/parparvm/com/codename1/backend/Web.java @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Outbound HTTP and HTTPS for server-side binaries. Backed by libcurl, so TLS + * verification, redirects, chunked decoding and the system certificate store all + * come from a library that is maintained for the purpose. + * + * Distinct from [Http], which is a raw-socket plaintext client for the host + * runtime's loopback control protocol. Use this one for anything real. + * + * **Certificate store.** A dynamically linked build finds the system CA bundle. + * A fully static build has whatever the image provides, which for a `scratch` + * container is nothing - and TLS then fails with "unable to get local issuer + * certificate". Ship a `ca-certificates.crt` and point curl at it with the + * `CURL_CA_BUNDLE` or `SSL_CERT_FILE` environment variable; both are read by + * libcurl itself, so no code here has to know about it. + */ +public final class Web { + + /** + * ASCII lower case, because String.toLowerCase() is LOCALE SENSITIVE and this + * platform has no Locale to ask for the root one. On a device set to Turkish + * the I of an ASCII token folds to a dotless i, so a header stored under one + * spelling is looked up under another and getHeader answers null: nothing is + * thrown, nothing is logged, and the caller reads a header that is there as + * absent. A header name is ASCII by specification. Copied rather than shared; + * see CLAUDE.md. Both arms of Web carry it, because both index headers. + */ + private static String asciiLower(String value) { + if(value == null) { + return null; + } + StringBuilder out = new StringBuilder(value.length()); + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + out.append(c >= 'A' && c <= 'Z' ? (char)(c + 32) : c); + } + return out.toString(); + } + + private Web() { + } + + /** An outbound response: status, body, and libcurl's message when it failed. */ + public static final class Result { + private final int status; + private final byte[] body; + private final String error; + private final Map headers; + + Result(int status, byte[] body, String error, Map headers) { + this.status = status; + this.body = body; + this.error = error; + this.headers = headers == null ? new LinkedHashMap() : headers; + } + + /** The HTTP status, or -1 when the transfer itself failed. */ + public int getStatus() { + return status; + } + + public boolean isSuccess() { + return status >= 200 && status < 300; + } + + public byte[] getBody() { + return body; + } + + public String getBodyAsString() { + if(body == null) { + return null; + } + try { + return new String(body, "UTF-8"); + } catch (IOException err) { + return new String(body); + } + } + + /** Non-null only when the transfer failed before producing a status. */ + public String getError() { + return error; + } + + /** + * The response headers, lower-cased names to values. + * + * A response's headers are half of what an API says -- the ETag S3 returns for + * a PUT, the content type of an object, the rate-limit budget a service + * publishes -- and a client that can only read the body cannot see any of it. + * Names are lower-cased because HTTP header names are case insensitive and a + * caller should not have to guess which case this server chose. + */ + public Map getHeaders() { + return headers; + } + + /** One header by name, matched case-insensitively. Null when absent. */ + public String getHeader(String name) { + return name == null ? null : (String)headers.get(asciiLower(name)); + } + } + + public static Result get(String url) throws IOException { + return request("GET", url, null, null); + } + + public static Result getJson(String url, String bearerToken) throws IOException { + List headers = new ArrayList(); + headers.add("Accept: application/json"); + if(bearerToken != null) { + headers.add("Authorization: Bearer " + bearerToken); + } + return request("GET", url, headers, null); + } + + public static Result postJson(String url, String json, String bearerToken) throws IOException { + List headers = new ArrayList(); + headers.add("Content-Type: application/json"); + headers.add("Accept: application/json"); + if(bearerToken != null) { + headers.add("Authorization: Bearer " + bearerToken); + } + byte[] payload; + try { + payload = json == null ? new byte[0] : json.getBytes("UTF-8"); + } catch (IOException err) { + throw new IOException("Could not encode the request body"); + } + return request("POST", url, headers, payload); + } + + /** + * - `headers`: a list of "Name: value" strings, or null + */ + public static Result request(String method, String url, List headers, byte[] body) throws IOException { + if(url == null) { + throw new IOException("No URL"); + } + StringBuilder joined = new StringBuilder(); + if(headers != null) { + for(int iter = 0 ; iter < headers.size() ; iter++) { + if(iter > 0) { + joined.append('\n'); + } + joined.append(String.valueOf(headers.get(iter))); + } + } + long handle = performImpl(method, url, joined.toString(), body); + if(handle == 0) { + throw new IOException("Could not start a request to " + url); + } + try { + int status = statusImpl(handle); + String error = errorImpl(handle); + if(status < 0) { + throw new IOException("Request to " + url + " failed: " + + (error == null ? "unknown error" : error)); + } + return new Result(status, bodyImpl(handle), error, + parseHeaders(headersImpl(handle))); + } finally { + freeImpl(handle); + } + } + + /** + * libcurl hands back the raw header block, status lines and all. Redirects + * mean there can be several blocks; the LAST one describes the response the + * caller got, so a later block replaces an earlier one rather than merging + * with it. + */ + static Map parseHeaders(String raw) { + Map out = new LinkedHashMap(); + if(raw == null) { + return out; + } + int at = 0; + while(at < raw.length()) { + int end = raw.indexOf('\n', at); + if(end < 0) { + end = raw.length(); + } + String line = raw.substring(at, end).trim(); + at = end + 1; + if(line.length() == 0) { + continue; + } + if(line.regionMatches(true, 0, "HTTP/", 0, 5)) { + // A new status line: everything before it belonged to a redirect. + out.clear(); + continue; + } + int colon = line.indexOf(':'); + if(colon <= 0) { + continue; + } + out.put(asciiLower(line.substring(0, colon).trim()), + line.substring(colon + 1).trim()); + } + return out; + } + + private static native long performImpl(String method, String url, String headerLines, byte[] body); + private static native String headersImpl(long handle); + private static native int statusImpl(long handle); + private static native String errorImpl(long handle); + private static native byte[] bodyImpl(long handle); + private static native void freeImpl(long handle); +} diff --git a/vm/backend/native/cn1_backend_crypto.c b/vm/backend/native/cn1_backend_crypto.c new file mode 100644 index 00000000000..7e978a701c9 --- /dev/null +++ b/vm/backend/native/cn1_backend_crypto.c @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * The crypto a server needs to authenticate a request: SHA-256, HMAC-SHA-256, + * PBKDF2 and a source of randomness that is actually random. + * + * All four come from OpenSSL, which the backend already links for outbound TLS. + * None of them is written here. Hand-rolled HMAC and hand-rolled password hashing + * are the two most reliable ways to ship an authentication system that looks + * correct and is not, and a constant-time comparison written in Java would be + * compiled into something that is not constant time. + */ +#include "cn1_globals.h" +#include +#include +#ifndef _WIN32 +#include /* CN1_RESUME_THREAD expands to usleep */ +#endif +#include +#include +#include +#include +#include +#include + +static JAVA_OBJECT cn1BytesToArray(CODENAME_ONE_THREAD_STATE, const unsigned char* data, int length) { + JAVA_OBJECT arr = allocArray(threadStateData, length, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); + if(length > 0 && data != NULL) { + memcpy((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)arr)->data, data, (size_t)length); + } + return arr; +} + +JAVA_OBJECT com_codename1_backend_Crypto_sha256Impl___byte_1ARRAY_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT data) { + unsigned char digest[SHA256_DIGEST_LENGTH]; + JAVA_ARRAY arr; + if(data == JAVA_NULL) { + return JAVA_NULL; + } + arr = (JAVA_ARRAY)data; + SHA256((const unsigned char*)(JAVA_ARRAY_BYTE*)arr->data, (size_t)arr->length, digest); + return cn1BytesToArray(threadStateData, digest, SHA256_DIGEST_LENGTH); +} + +/* + * SHA-1 and MD5 are here for one reason: the database wire protocols specify + * them. MySQL's mysql_native_password is SHA1-based and PostgreSQL's md5 method + * is MD5-based, and a client that refuses them cannot talk to the servers that + * are deployed. Neither is used for anything this code chooses -- passwords go + * through PBKDF2 and tokens through HMAC-SHA-256. + */ +JAVA_OBJECT com_codename1_backend_Crypto_sha1Impl___byte_1ARRAY_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT data) { + unsigned char digest[SHA_DIGEST_LENGTH]; + JAVA_ARRAY arr; + if(data == JAVA_NULL) { + return JAVA_NULL; + } + arr = (JAVA_ARRAY)data; + SHA1((const unsigned char*)(JAVA_ARRAY_BYTE*)arr->data, (size_t)arr->length, digest); + return cn1BytesToArray(threadStateData, digest, SHA_DIGEST_LENGTH); +} + +JAVA_OBJECT com_codename1_backend_Crypto_md5Impl___byte_1ARRAY_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT data) { + unsigned char digest[MD5_DIGEST_LENGTH]; + JAVA_ARRAY arr; + if(data == JAVA_NULL) { + return JAVA_NULL; + } + arr = (JAVA_ARRAY)data; + MD5((const unsigned char*)(JAVA_ARRAY_BYTE*)arr->data, (size_t)arr->length, digest); + return cn1BytesToArray(threadStateData, digest, MD5_DIGEST_LENGTH); +} + +JAVA_OBJECT com_codename1_backend_Crypto_hmacSha256Impl___byte_1ARRAY_byte_1ARRAY_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT key, JAVA_OBJECT data) { + unsigned char mac[EVP_MAX_MD_SIZE]; + unsigned int macLength = 0; + JAVA_ARRAY keyArr; + JAVA_ARRAY dataArr; + if(key == JAVA_NULL || data == JAVA_NULL) { + return JAVA_NULL; + } + keyArr = (JAVA_ARRAY)key; + dataArr = (JAVA_ARRAY)data; + if(HMAC(EVP_sha256(), + (const void*)(JAVA_ARRAY_BYTE*)keyArr->data, (int)keyArr->length, + (const unsigned char*)(JAVA_ARRAY_BYTE*)dataArr->data, (size_t)dataArr->length, + mac, &macLength) == NULL) { + return JAVA_NULL; + } + return cn1BytesToArray(threadStateData, mac, (int)macLength); +} + +JAVA_OBJECT com_codename1_backend_Crypto_pbkdf2Impl___byte_1ARRAY_byte_1ARRAY_int_int_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT password, JAVA_OBJECT salt, JAVA_INT iterations, JAVA_INT length) { + JAVA_ARRAY pw; + JAVA_ARRAY sl; + unsigned char* out; + JAVA_OBJECT result; + if(password == JAVA_NULL || salt == JAVA_NULL || length <= 0 || iterations <= 0) { + return JAVA_NULL; + } + pw = (JAVA_ARRAY)password; + sl = (JAVA_ARRAY)salt; + out = (unsigned char*)malloc((size_t)length); + if(out == NULL) { + return JAVA_NULL; + } + CN1_YIELD_THREAD; /* deliberately slow; do not stall the collector on it */ + if(PKCS5_PBKDF2_HMAC((const char*)(JAVA_ARRAY_BYTE*)pw->data, (int)pw->length, + (const unsigned char*)(JAVA_ARRAY_BYTE*)sl->data, (int)sl->length, + (int)iterations, EVP_sha256(), (int)length, out) != 1) { + CN1_RESUME_THREAD; + free(out); + return JAVA_NULL; + } + CN1_RESUME_THREAD; + result = cn1BytesToArray(threadStateData, out, length); + OPENSSL_cleanse(out, (size_t)length); + free(out); + return result; +} + +/* + * Cryptographically secure randomness, not java.util.Random. A session token or a + * salt drawn from a predictable generator is a forgeable one. + */ +JAVA_OBJECT com_codename1_backend_Crypto_randomBytesImpl___int_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_INT length) { + unsigned char* out; + JAVA_OBJECT result; + if(length <= 0) { + return JAVA_NULL; + } + out = (unsigned char*)malloc((size_t)length); + if(out == NULL) { + return JAVA_NULL; + } + if(RAND_bytes(out, (int)length) != 1) { + /* Never fall back to a weaker source: a caller that gets bytes assumes they + are unpredictable, and there is no way to signal "these are not". */ + free(out); + return JAVA_NULL; + } + result = cn1BytesToArray(threadStateData, out, length); + OPENSSL_cleanse(out, (size_t)length); + free(out); + return result; +} + +/* + * Constant-time comparison. In Java this would be compiled into whatever the + * optimizer likes, and an early exit on the first differing byte leaks the prefix + * length of a guess -- which is enough to forge a MAC one byte at a time. + */ +JAVA_BOOLEAN com_codename1_backend_Crypto_equalsConstantTimeImpl___byte_1ARRAY_byte_1ARRAY_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT a, JAVA_OBJECT b) { + JAVA_ARRAY aa; + JAVA_ARRAY bb; + if(a == JAVA_NULL || b == JAVA_NULL) { + return JAVA_FALSE; + } + aa = (JAVA_ARRAY)a; + bb = (JAVA_ARRAY)b; + if(aa->length != bb->length) { + return JAVA_FALSE; + } + return CRYPTO_memcmp((const void*)(JAVA_ARRAY_BYTE*)aa->data, + (const void*)(JAVA_ARRAY_BYTE*)bb->data, + (size_t)aa->length) == 0 ? JAVA_TRUE : JAVA_FALSE; +} diff --git a/vm/backend/native/cn1_backend_db.c b/vm/backend/native/cn1_backend_db.c new file mode 100644 index 00000000000..e349e5e341d --- /dev/null +++ b/vm/backend/native/cn1_backend_db.c @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * SQLite persistence for server-side binaries, straight onto the engine the + * translator already bundles (-Dcn1.sqlite=true drops cn1_sqlite3.c and the + * amalgamation into the source root). Not com.codename1.db.Database, which routes + * every call through CodenameOneImplementation. + * + * Handles are pointers cast to JAVA_LONG; 0 means "not open", so a failed open + * needs nothing freed. The prepare/step/finalize cycle is exposed rather than + * hidden behind an exec-string, because parameter binding is what keeps user data + * out of the SQL text. + */ +#include "cn1_globals.h" +#include +#include +#ifndef _WIN32 +#include /* CN1_RESUME_THREAD expands to usleep */ +#endif +/* + * Built as stubs when the engine is left out (CN1_BACKEND_SQLITE=0), rather than + * dropped from the build. A native whose C symbol is absent takes its JAVA method + * with it -- see BytecodeMethod.isMethodUsedByNative -- so removing this file + * would make Db.open link fine and do nothing. openImpl returning 0 is the "could + * not open" answer Db already handles, so a program built without the engine gets + * a clean IOException naming the path instead of silence. + */ +#ifdef CN1_BACKEND_NO_SQLITE + +JAVA_LONG com_codename1_backend_Db_openImpl___java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT path) { + return 0; +} + +JAVA_INT com_codename1_backend_Db_closeImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + return 0; +} + +JAVA_OBJECT com_codename1_backend_Db_errorImpl___long_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + return newStringFromCString(threadStateData, + "this binary was built without SQLite (CN1_BACKEND_SQLITE=0)"); +} + +JAVA_LONG com_codename1_backend_Db_prepareImpl___long_java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT sql) { + return 0; +} + +JAVA_INT com_codename1_backend_Db_bindStringImpl___long_int_java_lang_String_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index, JAVA_OBJECT value) { + return 0; +} + +JAVA_INT com_codename1_backend_Db_bindLongImpl___long_int_long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index, JAVA_LONG value) { + return 0; +} + +JAVA_INT com_codename1_backend_Db_bindDoubleImpl___long_int_double_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index, JAVA_DOUBLE value) { + return 0; +} + +JAVA_INT com_codename1_backend_Db_bindNullImpl___long_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index) { + return 0; +} + +JAVA_INT com_codename1_backend_Db_bindBlobImpl___long_int_byte_1ARRAY_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index, JAVA_OBJECT value) { + return 0; +} + +JAVA_INT com_codename1_backend_Db_stepImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt) { + return -1; +} + +JAVA_INT com_codename1_backend_Db_columnCountImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt) { + return 0; +} + +JAVA_OBJECT com_codename1_backend_Db_columnNameImpl___long_int_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index) { + return JAVA_NULL; +} + +JAVA_INT com_codename1_backend_Db_columnTypeImpl___long_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index) { + return 5; /* TYPE_NULL */ +} + +JAVA_OBJECT com_codename1_backend_Db_columnStringImpl___long_int_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index) { + return JAVA_NULL; +} + +JAVA_LONG com_codename1_backend_Db_columnLongImpl___long_int_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index) { + return 0; +} + +JAVA_DOUBLE com_codename1_backend_Db_columnDoubleImpl___long_int_R_double(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index) { + return 0; +} + +JAVA_OBJECT com_codename1_backend_Db_columnBlobImpl___long_int_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt, JAVA_INT index) { + return JAVA_NULL; +} + +JAVA_VOID com_codename1_backend_Db_finalizeImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmt) { +} + +JAVA_INT com_codename1_backend_Db_changesImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + return 0; +} + +JAVA_LONG com_codename1_backend_Db_lastInsertRowIdImpl___long_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + return 0; +} + +#else + +#include "cn1_sqlite3.h" + +JAVA_LONG com_codename1_backend_Db_openImpl___java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT path) { + sqlite3* db = NULL; + const char* p = path == JAVA_NULL ? NULL : stringToUTF8(threadStateData, path); + if(p == NULL) { + return 0; + } + if(sqlite3_open(p, &db) != SQLITE_OK) { + /* sqlite3_open allocates a handle even on failure so the error can be read; + close it here rather than leaking one per failed open. */ + if(db != NULL) { + sqlite3_close(db); + } + return 0; + } + return (JAVA_LONG)(intptr_t)db; +} + +JAVA_INT com_codename1_backend_Db_closeImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + sqlite3* db = (sqlite3*)(intptr_t)handle; + if(db == NULL) { + return 0; + } + return sqlite3_close(db) == SQLITE_OK ? 0 : -1; +} + +JAVA_OBJECT com_codename1_backend_Db_errorImpl___long_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + sqlite3* db = (sqlite3*)(intptr_t)handle; + const char* msg = db == NULL ? "database is not open" : sqlite3_errmsg(db); + return msg == NULL ? JAVA_NULL : newStringFromCString(threadStateData, msg); +} + +JAVA_LONG com_codename1_backend_Db_prepareImpl___long_java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT sql) { + sqlite3* db = (sqlite3*)(intptr_t)handle; + sqlite3_stmt* stmt = NULL; + const char* text = sql == JAVA_NULL ? NULL : stringToUTF8(threadStateData, sql); + if(db == NULL || text == NULL) { + return 0; + } + if(sqlite3_prepare_v2(db, text, -1, &stmt, NULL) != SQLITE_OK) { + return 0; + } + return (JAVA_LONG)(intptr_t)stmt; +} + +JAVA_INT com_codename1_backend_Db_bindStringImpl___long_int_java_lang_String_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index, JAVA_OBJECT value) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + if(stmt == NULL) { + return SQLITE_MISUSE; + } + if(value == JAVA_NULL) { + return sqlite3_bind_null(stmt, index); + } + /* SQLITE_TRANSIENT: the scratch buffer stringToUTF8 returns is reused by the + next conversion on this thread, so sqlite must take its own copy. */ + return sqlite3_bind_text(stmt, index, stringToUTF8(threadStateData, value), -1, + SQLITE_TRANSIENT); +} + +JAVA_INT com_codename1_backend_Db_bindLongImpl___long_int_long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index, JAVA_LONG value) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + if(stmt == NULL) { + return SQLITE_MISUSE; + } + return sqlite3_bind_int64(stmt, index, (sqlite3_int64)value); +} + +JAVA_INT com_codename1_backend_Db_bindDoubleImpl___long_int_double_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index, JAVA_DOUBLE value) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + if(stmt == NULL) { + return SQLITE_MISUSE; + } + return sqlite3_bind_double(stmt, index, value); +} + +JAVA_INT com_codename1_backend_Db_bindNullImpl___long_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + if(stmt == NULL) { + return SQLITE_MISUSE; + } + return sqlite3_bind_null(stmt, index); +} + +/* 1 = a row is available, 0 = finished, -1 = error. */ +JAVA_INT com_codename1_backend_Db_stepImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + int rc; + if(stmt == NULL) { + return -1; + } + CN1_YIELD_THREAD; + rc = sqlite3_step(stmt); + CN1_RESUME_THREAD; + if(rc == SQLITE_ROW) { + return 1; + } + if(rc == SQLITE_DONE) { + return 0; + } + return -1; +} + +JAVA_INT com_codename1_backend_Db_columnCountImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + return stmt == NULL ? 0 : sqlite3_column_count(stmt); +} + +JAVA_OBJECT com_codename1_backend_Db_columnNameImpl___long_int_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + const char* name = stmt == NULL ? NULL : sqlite3_column_name(stmt, index); + return name == NULL ? JAVA_NULL : newStringFromCString(threadStateData, name); +} + +/* Mirrors SQLITE_INTEGER/FLOAT/TEXT/BLOB/NULL as 1..5. */ +JAVA_INT com_codename1_backend_Db_columnTypeImpl___long_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + return stmt == NULL ? 5 : sqlite3_column_type(stmt, index); +} + +JAVA_OBJECT com_codename1_backend_Db_columnStringImpl___long_int_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + const unsigned char* text = stmt == NULL ? NULL : sqlite3_column_text(stmt, index); + return text == NULL ? JAVA_NULL : newStringFromCString(threadStateData, (const char*)text); +} + +JAVA_LONG com_codename1_backend_Db_columnLongImpl___long_int_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + return stmt == NULL ? 0 : (JAVA_LONG)sqlite3_column_int64(stmt, index); +} + +JAVA_DOUBLE com_codename1_backend_Db_columnDoubleImpl___long_int_R_double(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + return stmt == NULL ? 0 : sqlite3_column_double(stmt, index); +} + +JAVA_INT com_codename1_backend_Db_bindBlobImpl___long_int_byte_1ARRAY_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index, JAVA_OBJECT value) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + JAVA_ARRAY arr; + if(stmt == NULL) { + return SQLITE_MISUSE; + } + if(value == JAVA_NULL) { + return sqlite3_bind_null(stmt, index); + } + arr = (JAVA_ARRAY)value; + /* SQLITE_TRANSIENT: sqlite copies, so the array may be collected or moved the + moment this returns. */ + return sqlite3_bind_blob(stmt, index, (const void*)(JAVA_ARRAY_BYTE*)arr->data, + (int)arr->length, SQLITE_TRANSIENT); +} + +JAVA_OBJECT com_codename1_backend_Db_columnBlobImpl___long_int_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle, JAVA_INT index) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + const void* data; + int length; + JAVA_OBJECT arr; + if(stmt == NULL) { + return JAVA_NULL; + } + /* sqlite3_column_bytes must be called AFTER sqlite3_column_blob: the blob call + is what performs any needed type conversion, and the length is only correct + once it has. */ + data = sqlite3_column_blob(stmt, index); + length = sqlite3_column_bytes(stmt, index); + if(data == NULL) { + length = 0; + } + arr = allocArray(threadStateData, length, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); + if(length > 0) { + memcpy((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)arr)->data, data, (size_t)length); + } + return arr; +} + +JAVA_VOID com_codename1_backend_Db_finalizeImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG stmtHandle) { + sqlite3_stmt* stmt = (sqlite3_stmt*)(intptr_t)stmtHandle; + if(stmt != NULL) { + sqlite3_finalize(stmt); + } +} + +JAVA_INT com_codename1_backend_Db_changesImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + sqlite3* db = (sqlite3*)(intptr_t)handle; + return db == NULL ? 0 : sqlite3_changes(db); +} + +JAVA_LONG com_codename1_backend_Db_lastInsertRowIdImpl___long_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + sqlite3* db = (sqlite3*)(intptr_t)handle; + return db == NULL ? 0 : (JAVA_LONG)sqlite3_last_insert_rowid(db); +} + +#endif /* CN1_BACKEND_NO_SQLITE */ diff --git a/vm/backend/native/cn1_backend_files.c b/vm/backend/native/cn1_backend_files.c new file mode 100644 index 00000000000..d9412d50d19 --- /dev/null +++ b/vm/backend/native/cn1_backend_files.c @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Static file serving, on the kernel's zero-copy path where there is one. + * + * sendfile() moves bytes from a file descriptor to a socket inside the kernel: + * no read into a user buffer, no write back out, and on Linux no copy at all for + * the page-cache pages. For a file server that is the difference between two + * copies per byte and none, and it is why this is worth a native rather than a + * read/write loop in Java. + * + * The signature differs between Linux and the BSDs -- Linux returns the count and + * advances an offset pointer, macOS takes the length by reference and reports how + * much it moved -- so both are wrapped behind one call that always returns bytes + * sent. + * + * There is deliberately no sendfile path for TLS: the whole point is that the + * kernel copies bytes it does not have to look at, and encrypted bytes have to be + * produced in user space. The Java side falls back to read + SSL_write there, and + * says so. + */ +#include "cn1_globals.h" +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#if defined(__linux__) +#include +#include +/* From linux/openat2.h. Defined here so the build does not require a kernel header + that older distributions ship without. */ +#define CN1_RESOLVE_BENEATH 0x08 +#endif +#include +#include +#endif + +#if defined(__linux__) +#include +#define CN1_HAVE_SENDFILE 1 +#elif defined(__APPLE__) || defined(__FreeBSD__) +#include +#include +#define CN1_HAVE_SENDFILE 1 +#endif + +/* Opens for reading. Returns the descriptor, or -1. */ +/* + * Opens a path under `root` and refuses anything that resolves outside it, in one + * syscall that the filesystem cannot race. + * + * open-then-realPath cannot do this. The check runs against a SECOND lookup, so a + * symlink under a writable document root can point outside for the open and inside + * for the check, and the descriptor that gets served is the outside file. Comparing + * st_dev/st_ino afterwards narrows that window without closing it, because the + * second lookup is racy in the same way. + * + * RESOLVE_BENEATH makes the kernel refuse the escape during resolution instead, so + * there is no window to lose. Returns -2 where the kernel or platform has no + * openat2 -- the caller falls back to the older check rather than serving nothing. + */ +JAVA_INT com_codename1_backend_FileIo_openBeneathImpl___java_lang_String_java_lang_String_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT root, JAVA_OBJECT relative) { +#if defined(__linux__) && defined(SYS_openat2) + const char* rootPath; + const char* rel; + int dirFd; + int fd; + struct cn1_open_how { + uint64_t flags; + uint64_t mode; + uint64_t resolve; + } how; + if(root == JAVA_NULL || relative == JAVA_NULL) { + return -1; + } + rootPath = stringToUTF8(threadStateData, root); + if(rootPath == NULL) { + return -1; + } + dirFd = open(rootPath, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if(dirFd < 0) { + return -1; + } + rel = stringToUTF8(threadStateData, relative); + if(rel == NULL) { + close(dirFd); + return -1; + } + /* RESOLVE_BENEATH rejects an absolute path outright, and the caller's target + always starts at the root. */ + while(*rel == '/') { + rel++; + } + if(*rel == 0) { + close(dirFd); + return -1; + } + memset(&how, 0, sizeof(how)); + how.flags = (uint64_t)(O_RDONLY | O_CLOEXEC); + how.resolve = (uint64_t)CN1_RESOLVE_BENEATH; + fd = (int)syscall(SYS_openat2, dirFd, rel, &how, sizeof(how)); + close(dirFd); + if(fd < 0) { + /* An old kernel knows the number but not the struct, or does not know the + call at all. Either way this cannot answer, so say so rather than let the + caller read a refusal as "file missing". */ + if(errno == ENOSYS || errno == EINVAL || errno == E2BIG) { + return -2; + } + return -1; + } + return fd; +#else + (void)root; (void)relative; + return -2; +#endif +} + +JAVA_INT com_codename1_backend_FileIo_openReadImpl___java_lang_String_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT path) { +#ifdef _WIN32 + (void)path; + return -1; +#else + const char* p = path == JAVA_NULL ? NULL : stringToUTF8(threadStateData, path); + if(p == NULL) { + return -1; + } + return open(p, O_RDONLY | O_CLOEXEC); +#endif +} + +/* + * Fills out[0]=size, out[1]=modified-time-millis, out[2]=1 when it is a directory. + * One call rather than three so a request costs one stat, and taken from the OPEN + * DESCRIPTOR rather than the path: stat-then-open lets the file change underneath + * between the two, which is how a length header ends up disagreeing with the body. + */ +JAVA_INT com_codename1_backend_FileIo_statImpl___int_long_1ARRAY_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_OBJECT out) { +#ifdef _WIN32 + (void)fd; (void)out; + return -1; +#else + struct stat st; + JAVA_ARRAY_LONG* data; + if(fd < 0 || out == JAVA_NULL || ((JAVA_ARRAY)out)->length < 3) { + return -1; + } + if(fstat(fd, &st) != 0) { + return -1; + } + data = (JAVA_ARRAY_LONG*)((JAVA_ARRAY)out)->data; + data[0] = (JAVA_LONG)st.st_size; + /* Milliseconds, not whole seconds. StaticFiles builds its ETag from size and + this, so at one-second resolution a file replaced by different content of + the SAME size within the same second kept both halves of its validator and + every client holding the old ETag got a 304 for as long as it asked. The + Last-Modified header formats from the same value and is unaffected: HTTP + dates are whole seconds, so the extra precision is simply dropped there. */ +#if defined(__APPLE__) + data[1] = (JAVA_LONG)st.st_mtimespec.tv_sec * 1000LL + + (JAVA_LONG)(st.st_mtimespec.tv_nsec / 1000000L); +#elif defined(st_mtime) + /* POSIX.1-2008 defines st_mtime as a macro exactly when st_mtim exists. */ + data[1] = (JAVA_LONG)st.st_mtim.tv_sec * 1000LL + + (JAVA_LONG)(st.st_mtim.tv_nsec / 1000000L); +#else + data[1] = (JAVA_LONG)st.st_mtime * 1000LL; +#endif + data[2] = S_ISDIR(st.st_mode) ? 1 : 0; + return 0; +#endif +} + +/* + * Sends count bytes of inFd starting at offset straight to the socket. Returns how + * many moved, which may be fewer than asked -- the caller loops. -1 on error. + */ +/* + * Waits for the output socket to drain, bounded by its own send deadline. + * + * A copy of the one in cn1_backend_server.c rather than a shared symbol: it is + * fifteen lines and the two files are compiled independently. See that copy for + * why this waits instead of parking the virtual thread. + * + * Returns 1 when writable, 0 on timeout, -1 on error. + */ +static int cn1AwaitSocketWritable(int fd) { + struct pollfd waiting; + struct timeval tv; + socklen_t len = (socklen_t)sizeof(tv); + int timeout = -1; + int rc; + if(getsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char*)&tv, &len) == 0) { + long millis = (long)tv.tv_sec * 1000L + (long)(tv.tv_usec / 1000); + if(millis > 0) { + timeout = (int)millis; + } + } + waiting.fd = fd; + waiting.events = POLLOUT; + waiting.revents = 0; + do { + rc = poll(&waiting, 1, timeout); + } while(rc < 0 && errno == EINTR); + if(rc < 0) { + return -1; + } + return rc == 0 ? 0 : 1; +} + +JAVA_LONG com_codename1_backend_FileIo_sendFileImpl___int_int_long_long_R_long(CODENAME_ONE_THREAD_STATE, JAVA_INT outFd, JAVA_INT inFd, JAVA_LONG offset, JAVA_LONG count) { +#if defined(CN1_HAVE_SENDFILE) && defined(__linux__) + off_t off = (off_t)offset; + ssize_t n; + if(outFd < 0 || inFd < 0) { + return -1; + } + CN1_YIELD_THREAD; + for(;;) { + n = sendfile(outFd, inFd, &off, (size_t)count); + if(n >= 0) { + break; + } + if(errno == EINTR) { + continue; + } + if(errno == EAGAIN || errno == EWOULDBLOCK) { + /* The client's window is full, not an error. The descriptor is + non-blocking in virtual-thread mode, so this is the ordinary way a + large download to a slow client proceeds -- and reporting it as -1 + made StaticFiles close the connection and truncate the file. */ + int ready = cn1AwaitSocketWritable(outFd); + if(ready > 0) { + continue; + } + n = ready == 0 ? -3 : -1; + break; + } + n = -1; + break; + } + CN1_RESUME_THREAD; + return (JAVA_LONG)n; +#elif defined(CN1_HAVE_SENDFILE) + /* macOS/FreeBSD: len is in-out -- asked for on the way in, moved on the way + out -- and a partial send reports success with a smaller len, so a short + write is not an error here. */ + off_t len = (off_t)count; + int rc; + int sendErrno; + if(outFd < 0 || inFd < 0) { + return -1; + } + CN1_YIELD_THREAD; + for(;;) { + len = (off_t)count; + do { + rc = sendfile(inFd, outFd, (off_t)offset, &len, NULL, 0); + } while(rc < 0 && errno == EINTR); + /* Captured before CN1_RESUME_THREAD: the resume is a GC safepoint and can + park this thread on a timed wait, which overwrites errno. Read after + it, this classified a real sendfile failure by the WAIT's errno. */ + sendErrno = errno; + /* Anything except "the buffer was full and nothing moved" is an answer: + success, a real error, or a partial send the caller can advance on. */ + if(rc >= 0 || sendErrno != EAGAIN || len > 0) { + break; + } + /* EAGAIN having moved nothing is backpressure, and returning the 0 in + len made sendBody() read "no progress" as "the peer is gone" and drop + a large file mid-transfer for any client reading slower than the + server writes. That is the same truncation the Linux branch above + fixes; this twin kept it. Wait for the socket the same way. */ + if(cn1AwaitSocketWritable(outFd) <= 0) { + CN1_RESUME_THREAD; + return -1; + } + } + CN1_RESUME_THREAD; + if(rc < 0 && sendErrno != EAGAIN) { + return len > 0 ? (JAVA_LONG)len : -1; + } + return (JAVA_LONG)len; +#else + (void)outFd; (void)inFd; (void)offset; (void)count; + return -1; +#endif +} + +JAVA_BOOLEAN com_codename1_backend_FileIo_hasSendFileImpl___R_boolean(CODENAME_ONE_THREAD_STATE) { +#ifdef CN1_HAVE_SENDFILE + return JAVA_TRUE; +#else + return JAVA_FALSE; +#endif +} + +/* Plain read, for the TLS path and for platforms with no sendfile. */ +JAVA_INT com_codename1_backend_FileIo_readImpl___int_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { +#ifdef _WIN32 + (void)fd; (void)buffer; (void)offset; (void)length; + return -1; +#else + JAVA_ARRAY_BYTE* data; + ssize_t n; + if(fd < 0 || buffer == JAVA_NULL) { + return -1; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + CN1_YIELD_THREAD; + do { + n = read(fd, &data[offset], (size_t)length); + } while(n < 0 && errno == EINTR); + CN1_RESUME_THREAD; + return n < 0 ? -1 : (JAVA_INT)n; +#endif +} + +/* + * Resolves a path to its canonical form, following symlinks. The static handler + * uses this to prove a resolved file really is inside the document root -- a + * check on the request string alone is defeated by an encoded traversal or by a + * symlink pointing out of the tree. + */ +JAVA_OBJECT com_codename1_backend_FileIo_realPathImpl___java_lang_String_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT path) { +#ifdef _WIN32 + (void)path; + return JAVA_NULL; +#else + char resolved[4096]; + const char* p = path == JAVA_NULL ? NULL : stringToUTF8(threadStateData, path); + if(p == NULL) { + return JAVA_NULL; + } + if(realpath(p, resolved) == NULL) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, resolved); +#endif +} + +JAVA_VOID com_codename1_backend_FileIo_closeImpl___int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd) { +#ifndef _WIN32 + if(fd >= 0) { + close(fd); + } +#else + (void)fd; +#endif +} diff --git a/vm/backend/native/cn1_backend_http2.c b/vm/backend/native/cn1_backend_http2.c new file mode 100644 index 00000000000..4fb3c6afe12 --- /dev/null +++ b/vm/backend/native/cn1_backend_http2.c @@ -0,0 +1,1211 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * HTTP/2 on nghttp2. + * + * The framing layer is not written here and should not be. HPACK alone is a + * static table, a dynamic table with eviction, and Huffman coding, and getting any + * of it subtly wrong produces a connection that works until it does not. nghttp2 + * is the library curl already links, and it owns framing, HPACK, flow control, + * stream state, priority, CONTINUATION reassembly and GOAWAY. + * + * What is written here is the shape of the boundary. nghttp2 is callback-driven, + * but this deliberately does NOT call back into Java: a C callback reaching into + * the VM has to survive dead-code elimination and must not run while the collector + * is moving, and neither is worth arranging for a protocol adapter. Instead the + * callbacks accumulate COMPLETED requests into a queue on the session, and Java + * pulls from it. Data flows one way across the boundary at a time, and every + * native here is a plain function that returns. + */ +#include "cn1_globals.h" +#include +#include +#include +#include +#include +#include + +#define CN1_H2_MAX_HEADERS 64 +/* HttpServer.MAX_HEADER_BYTES. The count above bounds how MANY fields arrive, + never how large they are, so without this a peer stays under 64 fields and + still spends this process's memory a megabyte at a time -- across + CONTINUATION frames, and again on each stream its SETTINGS allows at once. + HTTP/1 has always refused that; this is the same ceiling for HTTP/2. */ +#define CN1_H2_MAX_HEADER_BYTES (64 * 1024) +/* And a ceiling across the whole session, for the same reason the body limit has + one: the per-stream figure is what ONE request may hold, and a client may hold + the advertised stream concurrency open at once without ever sending END_STREAM, + so the per-stream ceiling alone permits that multiple. Periodic control frames + keep such a connection alive indefinitely. */ +#define CN1_H2_MAX_SESSION_HEADER_BYTES (4 * CN1_H2_MAX_HEADER_BYTES) +/* The per-stream limit bounds ONE upload; it says nothing about how many run at + once. With the advertised concurrency a single connection could hold a hundred + nearly-complete 8 MiB bodies -- some 800 MiB of native buffers that live until + each stream completes or resets, and nothing stopped a second connection doing + the same. This is the ceiling for everything one session is holding. */ +#define CN1_H2_MAX_SESSION_BODY_BYTES (4 * CN1_H2_MAX_BODY_BYTES) +/* Mirrors HttpServer.MAX_BODY_BYTES: the HTTP/1 paths refuse a larger body and + HTTP/2 must agree, or the limit is only as good as the protocol chosen. */ +#define CN1_H2_MAX_BODY_BYTES (8 * 1024 * 1024) + +typedef struct CN1H2Header { + char* name; + char* value; +} CN1H2Header; + +typedef struct CN1H2Request { + int32_t streamId; + char* method; + char* path; + char* scheme; + char* authority; + CN1H2Header headers[CN1_H2_MAX_HEADERS]; + int headerCount; + size_t headerBytes; + unsigned char* body; + size_t bodyLength; + size_t bodyCapacity; + int complete; + struct CN1H2Request* next; +} CN1H2Request; + +/* + * One response body, owned by the stream that is sending it. + * + * This used to be a single buffer on the session, and nghttp2 reads a body AFTER + * submit returns, while it pumps output. serveHttp2 submits every finished + * response before it drains, so with two streams completing in one receive cycle + * the second submit freed the first one's buffer and reset the shared offset: + * whichever body was submitted last got sent for both streams, or an empty one + * did. The provider's own source pointer is what nghttp2 offers for exactly this, + * and the session keeps the list so a stream reset before EOF still frees. + */ +typedef struct CN1H2Body { + int32_t streamId; + /* Exactly one of these carries the body. `data` is a buffer this owns; `fd` is + an open descriptor this owns and reads each frame out of, which is how a file + is served without its size ever existing in the heap. */ + unsigned char* data; + int fd; + int64_t fileOffset; + size_t length; + size_t offset; + struct CN1H2Body* next; +} CN1H2Body; + +typedef struct { + nghttp2_session* session; + /* Streams still being received, and requests ready for Java to take. */ + CN1H2Request* open; + CN1H2Request* readyHead; + CN1H2Request* readyTail; + CN1H2Request* current; /* the one Java is currently reading */ + /* Bytes nghttp2 wants written to the socket. Java drains this. */ + unsigned char* out; + size_t outLength; + size_t outCapacity; + /* Response bodies still being written, one per stream. See CN1H2Body. */ + struct CN1H2Body* bodies; +} CN1H2Session; + +/* + * Frees one body, closing the descriptor when that is what it holds. + * + * Separate from cn1H2ReleaseBody, which unlinks it first, so that teardown can walk + * the list straight through instead of searching it once per body -- and so that + * there is no way to free a body without closing its descriptor. Doing it inline in + * two places is how the descriptor leaked from the teardown path: a client that + * dropped the connection mid-download left one open per request. + */ +/* File-backed response bodies alive across ALL sessions. Descriptors are a + process resource, not a per-connection one: bounding them per session still + multiplies by the connection count, and running out stops the process + accepting sockets or opening files at all -- a failure with nothing to do + with whichever client caused it. Every such body is created in respondFile + and destroyed in cn1H2FreeBody, so those two are the whole accounting. */ +static _Atomic long cn1H2OpenFileBodies = 0; + +/* Heap held by submitted response bodies across ALL sessions, for the same + reason the descriptors are counted that way: a per-session limit is a limit + per CONNECTION, and the connection ceiling is in the thousands. Each session + pausing itself after one oversized body still lets the process hold that body + times every connection, which is gigabytes of native memory pinned by small + GET requests whose senders never open their windows. Maintained at the three + points that already exist -- submitted, drained, freed -- so it cannot drift + from what the bodies actually hold. */ +static _Atomic long cn1H2PendingBodyBytes = 0; + +/* The ceiling cn1H2PendingBodyBytes is reserved against, or 0 for none. + Kept here rather than passed per call so that the RESERVATION can sit next to + the allocation it bounds: a limit tested in Java and enforced in C is two + steps with a gap, and two sessions processed at once both read the counter + below the limit and then both allocate. Set once from Java at startup. */ +static _Atomic long cn1H2MaxBodyBytes = 0; + +/* The ceiling cn1H2OpenFileBodies is reserved against, or 0 for none. Same + reasoning as the byte ceiling above: tested in Java and taken in C is two + steps with a gap, so every worker finishing a file response at once passed + the check before any of them incremented, and the process-wide cap was really + the cap plus one per concurrent worker -- each holding a DESCRIPTOR. */ +static _Atomic long cn1H2MaxFileBodies = 0; + +/* Reserves one descriptor slot, atomically. Returns 0 when the ceiling is + reached, in which case nothing is taken. */ +static int cn1H2ReserveFileBody(void) { + long limit = atomic_load_explicit(&cn1H2MaxFileBodies, memory_order_relaxed); + long current = atomic_load_explicit(&cn1H2OpenFileBodies, memory_order_relaxed); + for(;;) { + if(limit > 0 && current + 1 > limit) { + return 0; + } + if(atomic_compare_exchange_weak_explicit(&cn1H2OpenFileBodies, ¤t, + current + 1, + memory_order_relaxed, + memory_order_relaxed)) { + return 1; + } + } +} + +/* Reserves `bytes` against the ceiling, atomically. Returns 0 when the + reservation would cross it, in which case nothing is added. */ +static int cn1H2ReserveBodyBytes(long bytes) { + long limit = atomic_load_explicit(&cn1H2MaxBodyBytes, memory_order_relaxed); + long current = atomic_load_explicit(&cn1H2PendingBodyBytes, memory_order_relaxed); + for(;;) { + if(limit > 0 && current + bytes > limit) { + return 0; + } + if(atomic_compare_exchange_weak_explicit(&cn1H2PendingBodyBytes, ¤t, + current + bytes, + memory_order_relaxed, + memory_order_relaxed)) { + return 1; + } + /* current now holds what another thread left; try again against that. */ + } +} + +/* And the INBOUND side, for the identical reason. The per-session ceilings below + bound one connection; the connection ceiling is in the thousands, so a few + clients holding streams just under their session limit still add up to the + whole machine. Counted where a request's bytes are added -- header fields and + body chunks -- and released in cn1H2FreeRequest, which is the one place a + request's memory goes away. */ +static _Atomic long cn1H2InboundBytes = 0; +/* The ceiling on that total. Four sessions' worth: enough that no honest client + meets it, small enough that a dishonest fleet cannot walk past it. */ +#define CN1_H2_MAX_PROCESS_INBOUND_BYTES (4 * (CN1_H2_MAX_SESSION_BODY_BYTES \ + + CN1_H2_MAX_SESSION_HEADER_BYTES)) + +static void cn1H2FreeBody(CN1H2Body* body) { + if(body->data != NULL && body->length > body->offset) { + atomic_fetch_sub_explicit(&cn1H2PendingBodyBytes, + (long)(body->length - body->offset), memory_order_relaxed); + } + if(body->fd >= 0) { + atomic_fetch_sub_explicit(&cn1H2OpenFileBodies, 1, memory_order_relaxed); + /* The descriptor became the session's when the response was submitted, so + this is the one place that closes it: at EOF, at an early stream reset, + and at teardown, all of which arrive here. */ + close(body->fd); + } + free(body->data); + free(body); +} + +static void cn1H2ReleaseBody(CN1H2Session* s, CN1H2Body* body) { + CN1H2Body** link = &s->bodies; + while(*link != NULL) { + if(*link == body) { + *link = body->next; + break; + } + link = &(*link)->next; + } + cn1H2FreeBody(body); +} + +static void cn1H2ReleaseBodyForStream(CN1H2Session* s, int32_t streamId) { + CN1H2Body* body = s->bodies; + while(body != NULL) { + CN1H2Body* next = body->next; + if(body->streamId == streamId) { + cn1H2ReleaseBody(s, body); + } + body = next; + } +} + +static CN1H2Request* cn1H2FindOpen(CN1H2Session* s, int32_t streamId) { + CN1H2Request* r = s->open; + while(r != NULL) { + if(r->streamId == streamId) { + return r; + } + r = r->next; + } + return NULL; +} + +static void cn1H2FreeRequest(CN1H2Request* r) { + int i; + if(r == NULL) { + return; + } + atomic_fetch_sub_explicit(&cn1H2InboundBytes, + (long)(r->bodyLength + r->headerBytes + sizeof(CN1H2Request)), + memory_order_relaxed); + free(r->method); + free(r->path); + free(r->scheme); + free(r->authority); + for(i = 0 ; i < r->headerCount ; i++) { + free(r->headers[i].name); + free(r->headers[i].value); + } + free(r->body); + free(r); +} + +static void cn1H2Unlink(CN1H2Request** list, CN1H2Request* target) { + CN1H2Request** link = list; + while(*link != NULL) { + if(*link == target) { + *link = target->next; + target->next = NULL; + return; + } + link = &(*link)->next; + } +} + +static void cn1H2Enqueue(CN1H2Session* s, CN1H2Request* r) { + r->next = NULL; + if(s->readyTail == NULL) { + s->readyHead = r; + s->readyTail = r; + } else { + s->readyTail->next = r; + s->readyTail = r; + } +} + +/* nghttp2 hands us bytes to put on the wire; they are buffered for Java to drain. */ +/* How much serialised output one session may hold before it has to be drained. + nghttp2 will happily fill this in one nghttp2_session_send() -- it emits as + much as the peer's flow-control window allows -- so a client that raises its + windows and asks for a large file could grow this buffer toward the whole + window before Java got a chance to write any of it out. Returning WOULDBLOCK + is the backpressure nghttp2 understands: it stops, keeps what it has not + handed over, and offers it again after the drain. */ +#define CN1_H2_MAX_OUT_BYTES (1024 * 1024) + +static ssize_t cn1H2Send(nghttp2_session* session, const uint8_t* data, size_t length, + int flags, void* userData) { + CN1H2Session* s = (CN1H2Session*)userData; + (void)session; + (void)flags; + if(s->outLength >= CN1_H2_MAX_OUT_BYTES) { + return NGHTTP2_ERR_WOULDBLOCK; + } + if(s->outLength + length > s->outCapacity) { + size_t grown = (s->outLength + length) * 2 + 4096; + unsigned char* buf = (unsigned char*)realloc(s->out, grown); + if(buf == NULL) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + s->out = buf; + s->outCapacity = grown; + } + memcpy(s->out + s->outLength, data, length); + s->outLength += length; + return (ssize_t)length; +} + +static int cn1H2OnBeginHeaders(nghttp2_session* session, const nghttp2_frame* frame, + void* userData) { + CN1H2Session* s = (CN1H2Session*)userData; + CN1H2Request* r; + (void)session; + if(frame->hd.type != NGHTTP2_HEADERS || frame->headers.cat != NGHTTP2_HCAT_REQUEST) { + return 0; + } + /* The STRUCTURE counts too, not only what arrives in it. It embeds + CN1_H2_MAX_HEADERS slots, so one is about a kilobyte before a single + header byte is read -- and a client that opens the advertised stream + concurrency and sends minimal headers keeps the payload counters near + zero while holding one of these per stream, per connection. Counted as + what it is: a fixed cost per open request, charged here and released in + cn1H2FreeRequest with everything else the request holds. */ + if(atomic_load_explicit(&cn1H2InboundBytes, memory_order_relaxed) + + (long)sizeof(CN1H2Request) > CN1_H2_MAX_PROCESS_INBOUND_BYTES) { + /* Refusing the stream rather than the connection: nghttp2 resets this + one and the peer's other streams carry on, which is the proportionate + answer to a process that is momentarily full. */ + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + r = (CN1H2Request*)calloc(1, sizeof(CN1H2Request)); + if(r == NULL) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + atomic_fetch_add_explicit(&cn1H2InboundBytes, (long)sizeof(CN1H2Request), + memory_order_relaxed); + r->streamId = frame->hd.stream_id; + r->next = s->open; + s->open = r; + return 0; +} + +static char* cn1H2Dup(const uint8_t* value, size_t length) { + char* out = (char*)malloc(length + 1); + if(out == NULL) { + return NULL; + } + memcpy(out, value, length); + out[length] = 0; + return out; +} + +static int cn1H2OnHeader(nghttp2_session* session, const nghttp2_frame* frame, + const uint8_t* name, size_t nameLen, + const uint8_t* value, size_t valueLen, + uint8_t flags, void* userData) { + CN1H2Session* s = (CN1H2Session*)userData; + CN1H2Request* r; + (void)session; + (void)flags; + if(frame->hd.type != NGHTTP2_HEADERS) { + return 0; + } + r = cn1H2FindOpen(s, frame->hd.stream_id); + if(r == NULL) { + return 0; + } + /* Charged before anything is duplicated, and charged for the pseudo-headers + too: a single enormous :path would otherwise walk straight past a ceiling + that only looked at ordinary fields. */ + r->headerBytes += (size_t)nameLen + (size_t)valueLen; + /* Charged in the SAME breath as r->headerBytes, and before every rejection + below, because cn1H2FreeRequest gives back r->headerBytes whichever way + this stream ends. Charging after the checks -- which is what this did -- + left the rejected field counted by headerBytes and never added to the + global, so the free subtracted bytes the global had never gained and the + total drifted DOWNWARD. Repeat a rejected header block and the process + cap stops being a cap at all, which is the opposite of what it is for. + The two figures have to move together or neither means anything. */ + atomic_fetch_add_explicit(&cn1H2InboundBytes, (long)(nameLen + valueLen), + memory_order_relaxed); + if(r->headerBytes > CN1_H2_MAX_HEADER_BYTES) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + { + /* The same walk the body limit does, and bounded the same way: the + streams are capped by the concurrency setting, and a field is capped + by the per-stream ceiling above, so this cannot become the expensive + part of parsing a header block. r is already on s->open, so its own + bytes are counted by the walk rather than added to it. */ + size_t total = 0; + CN1H2Request* other = s->open; + while(other != NULL) { + total += other->headerBytes; + other = other->next; + } + other = s->readyHead; + while(other != NULL) { + total += other->headerBytes; + other = other->next; + } + if(total > CN1_H2_MAX_SESSION_HEADER_BYTES) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + } + /* Already charged above, so this only asks whether the process is over. */ + if(atomic_load_explicit(&cn1H2InboundBytes, memory_order_relaxed) + > CN1_H2_MAX_PROCESS_INBOUND_BYTES) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + /* The pseudo-headers carry what a request line carries in HTTP/1.1. */ + if(nameLen == 7 && memcmp(name, ":method", 7) == 0) { + r->method = cn1H2Dup(value, valueLen); + return 0; + } + if(nameLen == 5 && memcmp(name, ":path", 5) == 0) { + r->path = cn1H2Dup(value, valueLen); + return 0; + } + if(nameLen == 7 && memcmp(name, ":scheme", 7) == 0) { + r->scheme = cn1H2Dup(value, valueLen); + return 0; + } + if(nameLen == 10 && memcmp(name, ":authority", 10) == 0) { + r->authority = cn1H2Dup(value, valueLen); + return 0; + } + if(nameLen > 0 && name[0] == ':') { + return 0; /* an unknown pseudo-header; nghttp2 has already validated it */ + } + if(r->headerCount >= CN1_H2_MAX_HEADERS) { + /* Reset the stream rather than keep the first CN1_H2_MAX_HEADERS and report + success, which handed the handler a request with a later cookie, + content-type or tracing header simply missing -- and nothing anywhere said + so. TEMPORAL_CALLBACK_FAILURE fails this one stream and leaves the + connection up; the client sees the request fail, which is the honest + answer and the closest thing HTTP/2 has to HTTP/1's 431. */ + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + r->headers[r->headerCount].name = cn1H2Dup(name, nameLen); + r->headers[r->headerCount].value = cn1H2Dup(value, valueLen); + r->headerCount++; + return 0; +} + +static int cn1H2OnData(nghttp2_session* session, uint8_t flags, int32_t streamId, + const uint8_t* data, size_t length, void* userData) { + CN1H2Session* s = (CN1H2Session*)userData; + CN1H2Request* r = cn1H2FindOpen(s, streamId); + (void)session; + (void)flags; + if(r == NULL) { + return 0; + } + /* The same ceiling both HTTP/1 framing paths enforce (HttpServer.MAX_BODY_BYTES). + Without it a peer can stream DATA on one stream -- or on each of the streams + its SETTINGS allows at once -- until this process is out of native memory, + which the HTTP/1 side simply does not permit. Resetting the stream rather + than failing the callback keeps the connection and its other streams alive: + one oversized upload is that request's problem, not the session's. */ + if(r->bodyLength + length > CN1_H2_MAX_BODY_BYTES) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + { + /* Walked rather than counted in a running total: the total would have to + be decremented everywhere a request is freed, and one missed path + leaks budget until the session refuses everything. Both lists hold + bodies -- open ones are still arriving, ready ones are waiting for + Java to read them -- and neither is longer than the concurrency + setting. r is on the open list, so its own bodyLength is already in + the sum and only the new bytes are added. */ + size_t total = length; + CN1H2Request* other = s->open; + while(other != NULL) { + total += other->bodyLength; + other = other->next; + } + other = s->readyHead; + while(other != NULL) { + total += other->bodyLength; + other = other->next; + } + if(total > CN1_H2_MAX_SESSION_BODY_BYTES) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + } + /* Tested before the append and charged after it, because cn1H2FreeRequest + gives back bodyLength: charging first would strand the bytes of an append + that then FAILS to grow the buffer, and the counter would drift up until + it refused everything. The load-then-add can overshoot when two sessions + cross together, by at most one chunk each, which is the right trade for a + coarse memory guard -- the alternative is a lock on the data path. */ + if(atomic_load_explicit(&cn1H2InboundBytes, memory_order_relaxed) + (long)length + > CN1_H2_MAX_PROCESS_INBOUND_BYTES) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + if(r->bodyLength + length > r->bodyCapacity) { + size_t grown = (r->bodyLength + length) * 2 + 1024; + if(grown > CN1_H2_MAX_BODY_BYTES) { + grown = CN1_H2_MAX_BODY_BYTES; + } + unsigned char* buf = (unsigned char*)realloc(r->body, grown); + if(buf == NULL) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + r->body = buf; + r->bodyCapacity = grown; + } + memcpy(r->body + r->bodyLength, data, length); + r->bodyLength += length; + atomic_fetch_add_explicit(&cn1H2InboundBytes, (long)length, memory_order_relaxed); + return 0; +} + +static int cn1H2OnFrameRecv(nghttp2_session* session, const nghttp2_frame* frame, + void* userData) { + CN1H2Session* s = (CN1H2Session*)userData; + CN1H2Request* r; + (void)session; + if((frame->hd.flags & NGHTTP2_FLAG_END_STREAM) == 0) { + return 0; + } + if(frame->hd.type != NGHTTP2_HEADERS && frame->hd.type != NGHTTP2_DATA) { + return 0; + } + r = cn1H2FindOpen(s, frame->hd.stream_id); + if(r == NULL) { + return 0; + } + /* The request is complete only now: END_STREAM is what says the client has + finished, whether it arrived on HEADERS or on the last DATA frame. */ + cn1H2Unlink(&s->open, r); + r->complete = 1; + cn1H2Enqueue(s, r); + return 0; +} + +static int cn1H2OnStreamClose(nghttp2_session* session, int32_t streamId, + uint32_t errorCode, void* userData) { + CN1H2Session* s = (CN1H2Session*)userData; + CN1H2Request* r = cn1H2FindOpen(s, streamId); + (void)session; + (void)errorCode; + if(r != NULL) { + /* Reset before it completed: drop it rather than leak the stream state. */ + cn1H2Unlink(&s->open, r); + cn1H2FreeRequest(r); + } else { + /* Not open, so it may be COMPLETE and waiting for Java to take it. Looking + only at the open list left a cancelled request queued: serveHttp2() then + ran its handler and tried to respond on a stream nghttp2 had already + closed, and that failure reached the outer catch and dropped the whole + connection -- resetting every other stream multiplexed on it. Not + s->current, which Java is reading right now; nextRequest frees that one + when it moves on. */ + CN1H2Request* ready = s->readyHead; + while(ready != NULL && ready->streamId != streamId) { + ready = ready->next; + } + if(ready != NULL) { + cn1H2Unlink(&s->readyHead, ready); + /* cn1H2Unlink does not know about the tail, and this may have BEEN the + tail. The list is bounded by the concurrency setting, so finding the + new one is cheaper than a second link to keep in step. */ + s->readyTail = s->readyHead; + while(s->readyTail != NULL && s->readyTail->next != NULL) { + s->readyTail = s->readyTail->next; + } + cn1H2FreeRequest(ready); + } + } + /* A response whose body nghttp2 never read to EOF -- the peer reset the stream, + or the body limit above reset it -- would otherwise sit on the list until the + session ends. */ + cn1H2ReleaseBodyForStream(s, streamId); + return 0; +} + +JAVA_LONG com_codename1_backend_Http2_createImpl___R_long(CODENAME_ONE_THREAD_STATE) { + nghttp2_session_callbacks* callbacks; + CN1H2Session* s; + nghttp2_settings_entry settings[2]; + + s = (CN1H2Session*)calloc(1, sizeof(CN1H2Session)); + if(s == NULL) { + return 0; + } + if(nghttp2_session_callbacks_new(&callbacks) != 0) { + free(s); + return 0; + } + nghttp2_session_callbacks_set_send_callback(callbacks, cn1H2Send); + nghttp2_session_callbacks_set_on_begin_headers_callback(callbacks, cn1H2OnBeginHeaders); + nghttp2_session_callbacks_set_on_header_callback(callbacks, cn1H2OnHeader); + nghttp2_session_callbacks_set_on_data_chunk_recv_callback(callbacks, cn1H2OnData); + nghttp2_session_callbacks_set_on_frame_recv_callback(callbacks, cn1H2OnFrameRecv); + nghttp2_session_callbacks_set_on_stream_close_callback(callbacks, cn1H2OnStreamClose); + + if(nghttp2_session_server_new(&s->session, callbacks, s) != 0) { + nghttp2_session_callbacks_del(callbacks); + free(s); + return 0; + } + nghttp2_session_callbacks_del(callbacks); + + /* The connection preface. A server MUST send SETTINGS first; a client that + does not see it will not proceed. */ + settings[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; + settings[0].value = 100; + settings[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; + settings[1].value = 1024 * 1024; + if(nghttp2_submit_settings(s->session, NGHTTP2_FLAG_NONE, settings, 2) != 0) { + nghttp2_session_del(s->session); + free(s); + return 0; + } + return (JAVA_LONG)(intptr_t)s; +} + +/* Feeds received bytes in. Returns how many were consumed, or -1. */ +JAVA_INT com_codename1_backend_Http2_receiveImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + JAVA_ARRAY_BYTE* data; + ssize_t n; + if(s == NULL || buffer == JAVA_NULL) { + return -1; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + n = nghttp2_session_mem_recv(s->session, (const uint8_t*)&data[offset], (size_t)length); + return n < 0 ? -1 : (JAVA_INT)n; +} + +/* Runs nghttp2's output side, filling the outbound buffer. */ +JAVA_INT com_codename1_backend_Http2_pumpImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + if(s == NULL) { + return -1; + } + return nghttp2_session_send(s->session) == 0 ? 0 : -1; +} + +JAVA_INT com_codename1_backend_Http2_pendingOutputImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + return s == NULL ? 0 : (JAVA_INT)s->outLength; +} + +/* + * Heap held by response bodies that have been SUBMITTED and not yet fully + * written. This is not outLength: that buffer is what nghttp2 has already + * serialised, while a submitted body is pulled from its provider only as the + * peer's flow-control window allows. A client that stops sending WINDOW_UPDATE + * therefore leaves every body it asked for retained here, which is the figure a + * caller has to cap -- flushing frees nothing when the window is shut. + * + * A file-backed body owns a descriptor rather than a buffer, so it adds no heap + * and is not counted; descriptors are bounded by the stream concurrency limit. + */ +JAVA_LONG com_codename1_backend_Http2_pendingBodyBytesImpl___long_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + CN1H2Body* body; + int64_t total = 0; + if(s == NULL) { + return 0; + } + for(body = s->bodies ; body != NULL ; body = body->next) { + if(body->data != NULL && body->length > body->offset) { + total += (int64_t)(body->length - body->offset); + } + } + return (JAVA_LONG)total; +} + +/* + * File-backed response bodies outstanding across the process. Reported + * separately from the byte figure because it is a different resource with a + * different limit: such a body holds a DESCRIPTOR and no heap, so it is + * invisible to the byte accounting, and a peer that never opens its window + * keeps one per stream for as long as it likes. + */ +/* + * Response-body heap outstanding across the PROCESS. The per-session figure says + * what one connection is holding; this says what the machine is holding, which + * is the number that decides whether there is memory left. + */ +JAVA_LONG com_codename1_backend_Http2_pendingBodyBytesAllImpl___R_long(CODENAME_ONE_THREAD_STATE) { + return (JAVA_LONG)atomic_load_explicit(&cn1H2PendingBodyBytes, memory_order_relaxed); +} + +JAVA_INT com_codename1_backend_Http2_pendingBodyFilesImpl___R_int(CODENAME_ONE_THREAD_STATE) { + return (JAVA_INT)atomic_load_explicit(&cn1H2OpenFileBodies, memory_order_relaxed); +} + +/* The ceiling for outstanding response bodies across the process. */ +JAVA_VOID com_codename1_backend_Http2_setMaxBodyBytesImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG limit) { + atomic_store_explicit(&cn1H2MaxBodyBytes, (long)limit, memory_order_relaxed); +} + +/* The ceiling for outstanding file-backed response bodies across the process. */ +JAVA_VOID com_codename1_backend_Http2_setMaxFileBodiesImpl___int(CODENAME_ONE_THREAD_STATE, JAVA_INT limit) { + atomic_store_explicit(&cn1H2MaxFileBodies, (long)limit, memory_order_relaxed); +} + +/* Takes everything nghttp2 wants written, and empties the buffer. */ +JAVA_OBJECT com_codename1_backend_Http2_drainImpl___long_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + JAVA_OBJECT arr; + if(s == NULL) { + return JAVA_NULL; + } + arr = allocArray(threadStateData, (int)s->outLength, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); + if(s->outLength > 0) { + memcpy((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)arr)->data, s->out, s->outLength); + s->outLength = 0; + } + /* Give the CAPACITY back too, not just the length. A session that once sent + something large otherwise keeps that buffer for as long as it stays open, + and a keep-alive pool of them holds every peak it ever reached. Shrunk to + the ordinary size, so the common case reallocates nothing. */ + if(s->outCapacity > CN1_H2_MAX_OUT_BYTES) { + unsigned char* shrunk = (unsigned char*)realloc(s->out, 8192); + if(shrunk != NULL) { + s->out = shrunk; + s->outCapacity = 8192; + } + } + return arr; +} + +/* + * Makes the next completed request current, so the accessors below describe it. + * Returns its stream id, or -1 when there is none waiting. + */ +JAVA_INT com_codename1_backend_Http2_nextRequestImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + if(s == NULL) { + return -1; + } + if(s->current != NULL) { + cn1H2FreeRequest(s->current); + s->current = NULL; + } + if(s->readyHead == NULL) { + return -1; + } + s->current = s->readyHead; + s->readyHead = s->readyHead->next; + if(s->readyHead == NULL) { + s->readyTail = NULL; + } + s->current->next = NULL; + return (JAVA_INT)s->current->streamId; +} + +JAVA_OBJECT com_codename1_backend_Http2_methodImpl___long_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + if(s == NULL || s->current == NULL || s->current->method == NULL) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, s->current->method); +} + +JAVA_OBJECT com_codename1_backend_Http2_pathImpl___long_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + if(s == NULL || s->current == NULL || s->current->path == NULL) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, s->current->path); +} + +JAVA_OBJECT com_codename1_backend_Http2_authorityImpl___long_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + if(s == NULL || s->current == NULL || s->current->authority == NULL) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, s->current->authority); +} + +JAVA_INT com_codename1_backend_Http2_headerCountImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + return (s == NULL || s->current == NULL) ? 0 : (JAVA_INT)s->current->headerCount; +} + +JAVA_OBJECT com_codename1_backend_Http2_headerNameImpl___long_int_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_INT index) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + if(s == NULL || s->current == NULL || index < 0 || index >= s->current->headerCount) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, s->current->headers[index].name); +} + +JAVA_OBJECT com_codename1_backend_Http2_headerValueImpl___long_int_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_INT index) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + if(s == NULL || s->current == NULL || index < 0 || index >= s->current->headerCount) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, s->current->headers[index].value); +} + +JAVA_OBJECT com_codename1_backend_Http2_bodyImpl___long_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + JAVA_OBJECT arr; + size_t length; + if(s == NULL || s->current == NULL) { + return JAVA_NULL; + } + length = s->current->bodyLength; + arr = allocArray(threadStateData, (int)length, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); + if(length > 0) { + memcpy((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)arr)->data, s->current->body, length); + } + return arr; +} + +/* nghttp2 reads the response body through this, after submit returns. */ +static ssize_t cn1H2ReadBody(nghttp2_session* session, int32_t streamId, uint8_t* buf, + size_t length, uint32_t* dataFlags, nghttp2_data_source* source, + void* userData) { + CN1H2Session* s = (CN1H2Session*)userData; + CN1H2Body* body = (CN1H2Body*)source->ptr; + size_t remaining; + (void)session; + (void)streamId; + if(body == NULL) { + *dataFlags |= NGHTTP2_DATA_FLAG_EOF; + return 0; + } + remaining = body->length - body->offset; + if(remaining > length) { + remaining = length; + } + if(remaining > 0) { + if(body->fd >= 0) { + /* Straight into nghttp2's frame buffer. pread rather than read so the + descriptor needs no seek position of its own -- two streams may be + serving the same file. */ + ssize_t got; + /* Retried here rather than deferred. NGHTTP2_ERR_DEFERRED suspends the + provider until nghttp2_session_resume_data() is called, and nothing + calls it -- a single EINTR would have left the download open and + silent forever. A regular file never returns EAGAIN, so an error that + is not EINTR is a real one. */ + do { + got = pread(body->fd, buf, remaining, + (off_t)(body->fileOffset + (int64_t)body->offset)); + } while(got < 0 && errno == EINTR); + if(got < 0) { + return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE; + } + if(got == 0) { + /* The file is shorter than Content-Length said -- it was truncated + under us. Ending the stream here sends fewer bytes than promised, + which the client detects; stalling forever would not. */ + *dataFlags |= NGHTTP2_DATA_FLAG_EOF; + cn1H2ReleaseBody(s, body); + return 0; + } + remaining = (size_t)got; + } else { + memcpy(buf, body->data + body->offset, remaining); + } + body->offset += remaining; + if(body->data != NULL) { + atomic_fetch_sub_explicit(&cn1H2PendingBodyBytes, (long)remaining, + memory_order_relaxed); + } + } + if(body->offset >= body->length) { + *dataFlags |= NGHTTP2_DATA_FLAG_EOF; + cn1H2ReleaseBody(s, body); + } + return (ssize_t)remaining; +} + +/* + * Builds the response header block shared by both response forms. + * + * The nva entries point INTO *statusOut and *headerOut, which the caller frees + * after submitting -- nghttp2 copies what it needs during the submit call. Returns + * the header count, or -1 when the status could not be read. + */ +static long cn1H2BuildHeaders(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT status, + JAVA_OBJECT headerLines, nghttp2_nv* nva, + char** statusOut, char** headerOut) { + char* statusCopy; + char* headerCopy = NULL; + size_t count = 0; + + *statusOut = NULL; + *headerOut = NULL; + { + const char* tmp = stringToUTF8(threadStateData, status); + if(tmp == NULL) { + return -1; + } + statusCopy = strdup(tmp); + if(statusCopy == NULL) { + return -1; + } + } + if(headerLines != JAVA_NULL) { + const char* tmp = stringToUTF8(threadStateData, headerLines); + if(tmp != NULL && tmp[0] != 0) { + headerCopy = strdup(tmp); + } + } + *statusOut = statusCopy; + *headerOut = headerCopy; + + nva[count].name = (uint8_t*)":status"; + nva[count].namelen = 7; + nva[count].value = (uint8_t*)statusCopy; + nva[count].valuelen = strlen(statusCopy); + nva[count].flags = NGHTTP2_NV_FLAG_NONE; + count++; + + if(headerCopy != NULL) { + char* line = headerCopy; + /* Counted first, because running out of room used to end the loop quietly: + the response went out with the headers that fit and reported success, so + a late Set-Cookie, a CORS header or a security header simply was not + there over HTTP/2 while HTTP/1 sent all of them. Refusing is the honest + answer -- the handler asked for something this path cannot deliver. */ + { + int wanted = count; + char* scan = headerCopy; + while(scan != NULL && *scan != 0) { + char* nl = strchr(scan, '\n'); + char* colon = strchr(scan, ':'); + /* The colon has to be on THIS line: strchr runs to the end of the + whole block, so a colon further down would have counted a line + that has none, and the count would refuse responses that fit. */ + if(colon != NULL && (nl == NULL || colon < nl)) { + wanted++; + } + scan = nl == NULL ? NULL : nl + 1; + } + if(wanted > CN1_H2_MAX_HEADERS) { + free(statusCopy); + free(headerCopy); + return -1; + } + } + while(line != NULL && *line != 0 && count < CN1_H2_MAX_HEADERS) { + char* nl = strchr(line, '\n'); + char* colon; + if(nl != NULL) { + *nl = 0; + } + colon = strchr(line, ':'); + if(colon != NULL) { + char* value = colon + 1; + *colon = 0; + while(*value == ' ') { + value++; + } + /* HTTP/2 header names must be lower case; a capital is a protocol + error the peer will reset the stream over. */ + { + char* c = line; + while(*c != 0) { + if(*c >= 'A' && *c <= 'Z') { + *c = (char)(*c - 'A' + 'a'); + } + c++; + } + } + /* Connection-specific headers are forbidden in HTTP/2. */ + if(strcmp(line, "connection") != 0 && strcmp(line, "keep-alive") != 0 + && strcmp(line, "transfer-encoding") != 0 && strcmp(line, "upgrade") != 0) { + nva[count].name = (uint8_t*)line; + nva[count].namelen = strlen(line); + nva[count].value = (uint8_t*)value; + nva[count].valuelen = strlen(value); + nva[count].flags = NGHTTP2_NV_FLAG_NONE; + count++; + } + } + line = nl == NULL ? NULL : nl + 1; + } + } + return (long)count; +} + +/* + * Submits a response. headerLines is "name: value" separated by '\n'; the status + * is passed separately because :status is a pseudo-header nghttp2 requires first. + */ +JAVA_INT com_codename1_backend_Http2_respondImpl___long_int_java_lang_String_java_lang_String_byte_1ARRAY_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_INT streamId, JAVA_OBJECT status, JAVA_OBJECT headerLines, JAVA_OBJECT body) { + CN1H2Body* pending; + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + nghttp2_nv nva[CN1_H2_MAX_HEADERS + 1]; + char* headerCopy = NULL; + char* statusCopy = NULL; + size_t count = 0; + nghttp2_data_provider provider; + int rc; + + if(s == NULL) { + return -1; + } + { + long built = cn1H2BuildHeaders(threadStateData, status, headerLines, nva, + &statusCopy, &headerCopy); + if(built < 0) { + return -1; + } + count = (size_t)built; + } + + /* A resubmission for the same stream would otherwise leave the old one to be + freed only at stream close. */ + cn1H2ReleaseBodyForStream(s, streamId); + pending = NULL; + if(body != JAVA_NULL && ((JAVA_ARRAY)body)->length > 0) { + JAVA_ARRAY arr = (JAVA_ARRAY)body; + /* RESERVED first. Charging after the copy spends exactly what the + ceiling exists to withhold, and does it once per session that happens + to be running -- so the real peak was the limit plus a body for every + concurrent responder, whatever the configured number said. */ + if(!cn1H2ReserveBodyBytes((long)arr->length)) { + free(statusCopy); + free(headerCopy); + return -2; + } + pending = (CN1H2Body*)malloc(sizeof(CN1H2Body)); + if(pending != NULL) { + pending->data = (unsigned char*)malloc((size_t)arr->length); + if(pending->data == NULL) { + free(pending); + pending = NULL; + } else { + memcpy(pending->data, (JAVA_ARRAY_BYTE*)arr->data, (size_t)arr->length); + pending->streamId = streamId; + pending->fd = -1; + pending->fileOffset = 0; + pending->length = (size_t)arr->length; + pending->offset = 0; + pending->next = s->bodies; + s->bodies = pending; + } + } + if(pending == NULL) { + /* The reservation outlived its body; give it back or the ceiling + ratchets down one failed allocation at a time. */ + atomic_fetch_sub_explicit(&cn1H2PendingBodyBytes, (long)arr->length, + memory_order_relaxed); + /* The body could not be copied. Submitting anyway sends the headers with + an EMPTY body and reports success, so the caller ships a 200 whose + content silently went missing under memory pressure. Failing here lets + it be seen. */ + free(statusCopy); + free(headerCopy); + return -1; + } + } + provider.source.ptr = pending; + provider.read_callback = cn1H2ReadBody; + + rc = nghttp2_submit_response(s->session, streamId, nva, count, + pending != NULL ? &provider : NULL); + free(statusCopy); + free(headerCopy); + return rc == 0 ? 0 : -1; +} + +/* + * Submits a response whose body is a range of an open file. + * + * The descriptor becomes the session's here, whatever happens: on the failure paths + * below and, once submitted, when the body is released at EOF, at an early stream + * reset or at teardown. A caller that closed it itself would pull the file out from + * under the provider mid-response. + */ +JAVA_INT com_codename1_backend_Http2_respondFileImpl___long_int_java_lang_String_java_lang_String_int_long_long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_INT streamId, JAVA_OBJECT status, JAVA_OBJECT headerLines, JAVA_INT fd, JAVA_LONG offset, JAVA_LONG length) { + CN1H2Body* pending; + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + nghttp2_nv nva[CN1_H2_MAX_HEADERS + 1]; + char* headerCopy = NULL; + char* statusCopy = NULL; + size_t count = 0; + nghttp2_data_provider provider; + int rc; + + if(s == NULL || fd < 0) { + if(fd >= 0) { + close(fd); + } + return -1; + } + { + long built = cn1H2BuildHeaders(threadStateData, status, headerLines, nva, + &statusCopy, &headerCopy); + if(built < 0) { + close(fd); + return -1; + } + count = (size_t)built; + } + + cn1H2ReleaseBodyForStream(s, streamId); + pending = (CN1H2Body*)malloc(sizeof(CN1H2Body)); + if(pending == NULL) { + close(fd); + free(statusCopy); + free(headerCopy); + return -1; + } + /* RESERVED before the descriptor is taken, not counted after. */ + if(!cn1H2ReserveFileBody()) { + free(pending); + free(statusCopy); + free(headerCopy); + return -2; + } + pending->streamId = streamId; + pending->data = NULL; + pending->fd = fd; + pending->fileOffset = (int64_t)offset; + pending->length = (size_t)length; + pending->offset = 0; + pending->next = s->bodies; + s->bodies = pending; + + provider.source.ptr = pending; + provider.read_callback = cn1H2ReadBody; + + rc = nghttp2_submit_response(s->session, streamId, nva, count, &provider); + if(rc != 0) { + /* The provider will never run, so nothing else will free this. */ + cn1H2ReleaseBody(s, pending); + } + free(statusCopy); + free(headerCopy); + return rc == 0 ? 0 : -1; +} + +JAVA_BOOLEAN com_codename1_backend_Http2_wantsMoreImpl___long_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + if(s == NULL) { + return JAVA_FALSE; + } + return (nghttp2_session_want_read(s->session) || nghttp2_session_want_write(s->session)) + ? JAVA_TRUE : JAVA_FALSE; +} + +JAVA_VOID com_codename1_backend_Http2_destroyImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1H2Session* s = (CN1H2Session*)(intptr_t)handle; + CN1H2Request* r; + if(s == NULL) { + return; + } + nghttp2_session_del(s->session); + r = s->open; + while(r != NULL) { + CN1H2Request* next = r->next; + cn1H2FreeRequest(r); + r = next; + } + r = s->readyHead; + while(r != NULL) { + CN1H2Request* next = r->next; + cn1H2FreeRequest(r); + r = next; + } + cn1H2FreeRequest(s->current); + free(s->out); + while(s->bodies != NULL) { + CN1H2Body* next = s->bodies->next; + cn1H2FreeBody(s->bodies); + s->bodies = next; + } + free(s); +} diff --git a/vm/backend/native/cn1_backend_net.c b/vm/backend/native/cn1_backend_net.c new file mode 100644 index 00000000000..6e1934e3985 --- /dev/null +++ b/vm/backend/native/cn1_backend_net.c @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Blocking TCP client sockets for the clean (server-side) target. + * + * This is deliberately NOT the Linux port's cn1_linux_socket.c: that one is + * reached through CodenameOneImplementation, which a server-side binary does not + * have. Same system calls, no platform layer. + * + * The handle is fd+1 rather than a heap struct, so 0 is "not connected" and + * nothing has to be freed on a failed connect -- a leak here would be a leak per + * request. + * + * Every blocking call is bracketed with CN1_YIELD_THREAD / CN1_RESUME_THREAD. + * Without that, a thread parked in recv() is a thread the concurrent collector + * cannot mark past, so one idle connection would stall GC for the whole process. + */ +#include "cn1_globals.h" +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#define CN1_CLOSE_SOCKET closesocket +typedef int cn1_socklen; +#else +#include +#include +#include +#include +#include +#include +#include +#define CN1_CLOSE_SOCKET close +typedef socklen_t cn1_socklen; +#endif + +static int cn1BackendFd(JAVA_LONG handle) { + return handle <= 0 ? -1 : (int)(handle - 1); +} + +static int cn1SetNonBlocking(int fd, int on) { +#ifdef _WIN32 + u_long mode = on ? 1 : 0; + return ioctlsocket(fd, FIONBIO, &mode) == 0 ? 0 : -1; +#else + int flags = fcntl(fd, F_GETFL, 0); + if(flags < 0) { + return -1; + } + flags = on ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK); + return fcntl(fd, F_SETFL, flags) == 0 ? 0 : -1; +#endif +} + +static int cn1ConnectPending(void) { +#ifdef _WIN32 + return WSAGetLastError() == WSAEWOULDBLOCK; +#else + return errno == EINPROGRESS; +#endif +} + +/* + * connect() that gives up when the caller said to. + * + * A blocking connect ignores the timeout entirely and waits out the OS TCP + * timeout, which is minutes when an address silently drops packets rather than + * refusing. A database URL's ten-second default then means nothing on the device + * while meaning exactly ten seconds in the JavaSE runtime, so the same + * misconfiguration looks like a slow start in development and a hung process in + * production. + * + * The socket goes back to blocking before returning: every read and write after + * this expects that. The deadline is per address, so a host resolving to several + * can take the timeout once for each -- which is the point, since the reachable + * one is usually not the first. + */ +/* MSG_NOSIGNAL where the platform has it, as the listener's write path uses. */ +#ifndef _WIN32 +#ifdef MSG_NOSIGNAL +#define CN1_OUT_SEND_FLAGS MSG_NOSIGNAL +#else +#define CN1_OUT_SEND_FLAGS 0 +#endif + +/* + * Ignores SIGPIPE, whose default action is to KILL the process. + * + * The listener does this when it binds, and Signals.installShutdownHandler does + * it too, but a packaged runtime need do neither: LambdaRuntime.run() only makes + * outbound connections. In that process a database or Runtime API peer that went + * away between one write and the next took the whole runtime down instead of + * raising an IOException. Done on connect because it has to precede any write, + * and it covers the TLS client as well -- SSL_write goes through write(2), where + * MSG_NOSIGNAL cannot reach. Idempotent, so calling it per connection is free. + */ +static void cn1IgnoreSigPipe(void) { + signal(SIGPIPE, SIG_IGN); +} +#else +#define CN1_OUT_SEND_FLAGS 0 +static void cn1IgnoreSigPipe(void) { +} +#endif + +static int cn1ConnectWithTimeout(int fd, const struct sockaddr* addr, cn1_socklen len, + int timeoutMillis) { + int err = 0; + cn1_socklen errLen = (cn1_socklen)sizeof(err); + int rc; + if(timeoutMillis <= 0 || cn1SetNonBlocking(fd, 1) != 0) { + /* No deadline asked for, or the socket refused to go non-blocking: the + blocking connect is still the right answer, just without a deadline. */ + return connect(fd, addr, len) == 0 ? 0 : -1; + } + rc = connect(fd, addr, len); + if(rc != 0) { + if(!cn1ConnectPending()) { + cn1SetNonBlocking(fd, 0); + return -1; + } +#ifdef _WIN32 + { + fd_set writable; + struct timeval tv; + FD_ZERO(&writable); + FD_SET((SOCKET)fd, &writable); + tv.tv_sec = timeoutMillis / 1000; + tv.tv_usec = (timeoutMillis % 1000) * 1000; + rc = select(0, 0, &writable, 0, &tv); + } +#else + { + /* poll rather than select: a server with many open connections hands + out descriptors above FD_SETSIZE, and select is undefined there. */ + struct pollfd waiting; + waiting.fd = fd; + waiting.events = POLLOUT; + waiting.revents = 0; + do { + rc = poll(&waiting, 1, timeoutMillis); + } while(rc < 0 && errno == EINTR); + } +#endif + if(rc <= 0) { + cn1SetNonBlocking(fd, 0); + return -1; /* timed out, or the wait itself failed */ + } + if(getsockopt(fd, SOL_SOCKET, SO_ERROR, (char*)&err, &errLen) != 0 || err != 0) { + cn1SetNonBlocking(fd, 0); + return -1; + } + } + cn1SetNonBlocking(fd, 0); + return 0; +} + +JAVA_LONG com_codename1_backend_Tcp_connectImpl___java_lang_String_int_int_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT host, JAVA_INT port, JAVA_INT timeoutMillis) { + struct addrinfo hints; + struct addrinfo* res = 0; + struct addrinfo* it; + char portStr[16]; + int fd = -1; + const char* h = host == JAVA_NULL ? 0 : stringToUTF8(threadStateData, host); + if(!h) { + return 0; + } + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + snprintf(portStr, sizeof(portStr), "%d", (int)port); + cn1IgnoreSigPipe(); + /* Yielded BEFORE the resolver, not after it. getaddrinfo blocks -- for the + full resolver timeout when DNS is slow or unreachable -- and the VM counted + this thread as running throughout, so a collection waited for it and every + unrelated request waited with it. The timeoutMillis argument does not cover + this either: it starts once there is an address to connect to. + + Safe across a yield because `h` is a copy in the thread state's own utf8 + buffer, which stringToUTF8 mallocs; nothing here holds a heap pointer. */ + CN1_YIELD_THREAD; + if(getaddrinfo(h, portStr, &hints, &res) != 0) { + CN1_RESUME_THREAD; + return 0; + } + for(it = res ; it != 0 ; it = it->ai_next) { + fd = (int)socket(it->ai_family, it->ai_socktype, it->ai_protocol); + if(fd < 0) { + continue; + } + if(cn1ConnectWithTimeout(fd, it->ai_addr, (cn1_socklen)it->ai_addrlen, + (int)timeoutMillis) == 0) { + break; + } + CN1_CLOSE_SOCKET(fd); + fd = -1; + } + CN1_RESUME_THREAD; + freeaddrinfo(res); + if(fd < 0) { + return 0; + } + return (JAVA_LONG)fd + 1; +} + +JAVA_INT com_codename1_backend_Tcp_readImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + int fd = cn1BackendFd(handle); + JAVA_ARRAY_BYTE* data; + long n; + if(fd < 0 || buffer == JAVA_NULL) { + return -2; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + /* + * This blocks, and on a virtual thread it blocks the HOST. + * + * CN1_YIELD_THREAD releases the thread to the COLLECTOR; it is not a park. The + * server's own readImpl parks on EAGAIN because its descriptor is registered in + * a host's poller, which is what resumes it. An outbound socket is in no poller, + * so there is nothing to wake it and yielding here would spin. + * + * The consequence is real: with one host per core, as many concurrent slow + * database reads as there are cores occupy every host, and unrelated HTTP + * connections stop being served. Making this park means giving outbound + * descriptors the same poller registration inbound ones have -- a scheduler + * feature, not a local change -- and until that exists the guide says so under + * "Limits worth knowing" rather than the mode quietly not holding. + */ + CN1_YIELD_THREAD; + n = (long)recv(fd, (char*)&data[offset], (size_t)length, 0); + CN1_RESUME_THREAD; + if(n == 0) { + return -1; /* orderly shutdown by the peer */ + } + if(n < 0) { + return -2; + } + return (JAVA_INT)n; +} + +JAVA_INT com_codename1_backend_Tcp_writeImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + int fd = cn1BackendFd(handle); + JAVA_ARRAY_BYTE* data; + JAVA_INT written = 0; + if(fd < 0 || buffer == JAVA_NULL) { + return -1; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + CN1_YIELD_THREAD; + /* send() may accept less than asked; loop so the Java side can treat a short + write as a hard failure rather than having to retry it itself. */ + while(written < length) { + long n = (long)send(fd, (const char*)&data[offset + written], + (size_t)(length - written), CN1_OUT_SEND_FLAGS); + if(n <= 0) { + CN1_RESUME_THREAD; + return -1; + } + written += (JAVA_INT)n; + } + CN1_RESUME_THREAD; + return written; +} + +JAVA_INT com_codename1_backend_Tcp_closeImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + int fd = cn1BackendFd(handle); + if(fd < 0) { + return 0; + } + return CN1_CLOSE_SOCKET(fd) == 0 ? 0 : -1; +} diff --git a/vm/backend/native/cn1_backend_server.c b/vm/backend/native/cn1_backend_server.c new file mode 100644 index 00000000000..0e003a32c54 --- /dev/null +++ b/vm/backend/native/cn1_backend_server.c @@ -0,0 +1,1201 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Listening sockets and a readiness poller for server-side binaries. + * + * Why a reactor rather than a thread per connection: a parked ParparVM thread was + * measured at 243KB on musl/arm64 (see vm/benchmarks ThreadCost), so ten thousand + * connections would be gigabytes of threads. A connection here is an fd; only the + * ones with a request in flight occupy a worker. + * + * epoll on Linux, kqueue on the BSDs and macOS, behind one interface. Level- + * triggered on purpose: edge-triggered requires draining every fd to EAGAIN on + * every wake-up, and the whole point of this design is that the poller hands a + * ready fd to a worker and stops thinking about it. + */ +#include "cn1_globals.h" +#include "cn1_virtual_thread.h" +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif +#include + +#ifndef _WIN32 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#if defined(__linux__) +#include +#define CN1_HAVE_EPOLL 1 +#elif defined(__APPLE__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) +#include +#include +#define CN1_HAVE_KQUEUE 1 +#endif + +/* Mirrors the Java side; keep in sync with Reactor. */ +#define CN1_EVENT_READ 1 +#define CN1_EVENT_WRITE 2 +#define CN1_EVENT_ONESHOT 4 + +#ifdef MSG_NOSIGNAL +#define CN1_SEND_FLAGS MSG_NOSIGNAL +#else +#define CN1_SEND_FLAGS 0 +#endif + +JAVA_INT com_codename1_backend_ServerSocket_bindImpl___java_lang_String_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT host, JAVA_INT port, JAVA_INT backlog) { +#ifdef _WIN32 + (void)host; (void)port; (void)backlog; + return -1; +#else + struct sockaddr_in addr; + int fd; + int on = 1; + const char* h = host == JAVA_NULL ? NULL : stringToUTF8(threadStateData, host); + + /* SIGPIPE's default action is to kill the process, and a client that goes away + mid-response makes send() raise it. Ignoring it here rather than only inside + Signals.installShutdownHandler(): that call is optional, so a server that + never made it died the first time a browser closed a tab. Setting it once at + bind costs nothing and cannot be skipped by a server that listens. */ + signal(SIGPIPE, SIG_IGN); + + if(h != NULL && h[0] != 0 && strcmp(h, "0.0.0.0") != 0) { + /* Resolved, not parsed as numeric IPv4. + inet_pton alone accepted only a dotted quad, so "localhost" -- the most + ordinary bind host there is -- and every IPv6 address failed startup, but + ONLY once packaged natively: the JavaSE side goes through + InetSocketAddress and takes all of them, so the configuration was proven + under cn1:backend and then would not start. */ + struct addrinfo hints; + struct addrinfo* res = NULL; + struct addrinfo* it; + char portStr[16]; + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; + snprintf(portStr, sizeof(portStr), "%d", (int)port); + /* The same reason the outbound connect yields around this: getaddrinfo + blocks, and a thread the VM believes is running holds up a collection + for as long as the resolver takes. `h` is in the thread state's own + malloc'd buffer, so it survives the yield. */ + CN1_YIELD_THREAD; + if(getaddrinfo(h, portStr, &hints, &res) != 0) { + CN1_RESUME_THREAD; + return -1; + } + CN1_RESUME_THREAD; + for(it = res ; it != NULL ; it = it->ai_next) { + fd = socket(it->ai_family, it->ai_socktype, it->ai_protocol); + if(fd < 0) { + continue; + } + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const char*)&on, sizeof(on)); + if(bind(fd, it->ai_addr, it->ai_addrlen) == 0 && listen(fd, backlog) == 0) { + freeaddrinfo(res); + return fd; + } + close(fd); + } + freeaddrinfo(res); + return -1; + } + + /* A null host means "every interface", and that has to include the v6 ones: + an AF_INET socket cannot accept an IPv6 client, so the default binding was + unreachable in an IPv6-only deployment while an explicitly named host -- + which takes the getaddrinfo path above -- worked. A v6 socket with + V6ONLY cleared serves both families through one descriptor. Falling back + to AF_INET keeps hosts with no IPv6 at all working exactly as before. */ + { + struct sockaddr_in6 addr6; + int off = 0; + fd = socket(AF_INET6, SOCK_STREAM, 0); + if(fd >= 0) { + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const char*)&on, sizeof(on)); + if(setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&off, + sizeof(off)) == 0) { + memset(&addr6, 0, sizeof(addr6)); + addr6.sin6_family = AF_INET6; + addr6.sin6_port = htons((unsigned short)port); + addr6.sin6_addr = in6addr_any; + if(bind(fd, (struct sockaddr*)&addr6, sizeof(addr6)) == 0 + && listen(fd, backlog) == 0) { + return fd; + } + } + close(fd); + } + } + + fd = socket(AF_INET, SOCK_STREAM, 0); + if(fd < 0) { + return -1; + } + /* Without SO_REUSEADDR a restart inside the TIME_WAIT window fails to bind, + which in a container is every restart. */ + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const char*)&on, sizeof(on)); + + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons((unsigned short)port); + addr.sin_addr.s_addr = htonl(INADDR_ANY); + if(bind(fd, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + close(fd); + return -1; + } + if(listen(fd, backlog) != 0) { + close(fd); + return -1; + } + return fd; +#endif +} + +/* The port actually bound, so a caller may ask for 0 and be told what it got. */ +/* + * A byte[] whose storage is a C buffer this file owns, handed to Java with no + * copy and never allocated by the collector. + * + * This works because of three properties of ParparVM that a moving or precise VM + * would not give us, all of them already true -- nothing about GC semantics is + * changed here: + * + * 1. `struct JavaArrayPrototype` holds `void* data` as a POINTER, separate from + * the header. allocArray happens to point it just past itself, but nothing + * requires that, so it can address a buffer the GC never allocated. + * 2. gcMarkObject validates a pointer against the page/extent tables BEFORE it + * dereferences anything, and returns for one that does not resolve. A header + * outside every heap page is therefore ignored rather than corrupted. + * 3. The sweep walks heap pages, so an object that is in none is never freed. + * + * The header is registered as an immortal root anyway. That is not needed to keep + * THIS array alive -- nothing sweeps it -- it is needed so the mark guard accepts + * the pointer, which is what lets a foreign array hold references that still get + * traced. A byte[] has no reference children, so for this one it is belt and + * braces; for the same trick applied to an object with fields it is load bearing. + * cn1AddImmortalRoot documents the off-heap registrant case explicitly. + * + * One per thread, allocated once and reused, so the steady-state allocation rate + * contributed by the read path is zero rather than small. + */ +static __thread struct JavaArrayPrototype* cn1BackendReadArray = 0; +static __thread char* cn1BackendReadStorage = 0; +static __thread JAVA_INT cn1BackendReadCap = 0; + +/* + * A zero-length array handed back to mean "nothing ready", kept SEPARATE from the + * read buffer above. + * + * The first version of this signalled by setting the read array's own length to + * zero, which corrupts anything still borrowing it: that header is one object per + * host thread, and a connection parsing out of it saw buffer.length become 0 + * underneath and died with an ArrayIndexOutOfBoundsException inside serveOne. The + * signal must not touch the buffer it is a signal about. + */ +static __thread struct JavaArrayPrototype* cn1BackendWouldBlockArray = 0; + +static struct JavaArrayPrototype* cn1BackendEnsureWouldBlockArray(void) { + if(cn1BackendWouldBlockArray == 0) { + cn1BackendWouldBlockArray = (struct JavaArrayPrototype*) + calloc(1, sizeof(struct JavaArrayPrototype)); + if(cn1BackendWouldBlockArray == 0) { + return 0; + } + cn1BackendWouldBlockArray->__codenameOneParentClsReference = &class_array1__JAVA_BYTE; + cn1BackendWouldBlockArray->__codenameOneGcMark = -1; + cn1BackendWouldBlockArray->__heapPosition = -1; + cn1BackendWouldBlockArray->dimensions = 1; + cn1BackendWouldBlockArray->primitiveSize = sizeof(JAVA_ARRAY_BYTE); + cn1BackendWouldBlockArray->length = 0; + cn1BackendWouldBlockArray->data = 0; + cn1AddImmortalRoot((JAVA_OBJECT)cn1BackendWouldBlockArray); + } + return cn1BackendWouldBlockArray; +} + +/* + * Whether awaitReadable probes with poll() before parking. Read once; see the + * discussion at the call site. 1 (probe) is the shipped default until the A/B + * on an idle host says otherwise. + */ +static int cn1BackendSpeculativePoll(void) { + static int cached = -1; + if(cached < 0) { + const char* v = getenv("CN1_HTTP_SPECULATIVE_POLL"); + cached = (v != 0 && v[0] == '0') ? 0 : 1; + } + return cached; +} + +/* + * The poller's event array, one per thread, grown on demand and reused. + * + * This ran as a malloc/free pair on EVERY poller wait. The read path above is + * pooled precisely "so the steady-state allocation rate contributed by the read + * path is zero rather than small", and the poller sits on the same loop -- once + * per scheduling turn, which under virtual threads is about once per request -- + * so it was contributing the allocation the read path had been taught not to. + * + * epoll and kqueue never both compile in (#if / #elif below), so one buffer with + * a byte capacity serves whichever is built, and the entry size is passed in + * rather than baked in. + */ +static __thread void* cn1BackendEventBuf = 0; +static __thread int cn1BackendEventCap = 0; + +static void* cn1BackendEnsureEventBuf(int capacity, size_t entrySize) { + void* grown; + if(capacity <= 0) { + return 0; + } + if(cn1BackendEventBuf != 0 && cn1BackendEventCap >= capacity) { + return cn1BackendEventBuf; + } + grown = realloc(cn1BackendEventBuf, entrySize * (size_t)capacity); + if(grown == 0) { + return 0; + } + cn1BackendEventBuf = grown; + cn1BackendEventCap = capacity; + return grown; +} + +static struct JavaArrayPrototype* cn1BackendEnsureReadArray(JAVA_INT capacity) { + if(capacity <= 0) { + return 0; + } + if(cn1BackendReadArray != 0 && cn1BackendReadCap >= capacity) { + return cn1BackendReadArray; + } + if(cn1BackendReadArray == 0) { + cn1BackendReadArray = (struct JavaArrayPrototype*) + calloc(1, sizeof(struct JavaArrayPrototype)); + if(cn1BackendReadArray == 0) { + return 0; + } + cn1BackendReadArray->__codenameOneParentClsReference = &class_array1__JAVA_BYTE; + cn1BackendReadArray->__codenameOneGcMark = -1; + cn1BackendReadArray->__heapPosition = -1; + cn1BackendReadArray->dimensions = 1; + cn1BackendReadArray->primitiveSize = sizeof(JAVA_ARRAY_BYTE); + cn1AddImmortalRoot((JAVA_OBJECT)cn1BackendReadArray); + } + { + char* grown = (char*)realloc(cn1BackendReadStorage, (size_t)capacity); + if(grown == 0) { + return 0; + } + cn1BackendReadStorage = grown; + cn1BackendReadCap = capacity; + // The header outlives every grow, so the Java side keeps one identity and + // only the storage moves -- which is safe precisely because no Java + // reference points INTO the storage, only at the header. + cn1BackendReadArray->data = cn1BackendReadStorage; + } + return cn1BackendReadArray; +} + +JAVA_OBJECT com_codename1_backend_ServerSocket_threadReadBufferImpl___int_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_INT capacity) { + struct JavaArrayPrototype* a = cn1BackendEnsureReadArray(capacity); + if(a == 0) { + return JAVA_NULL; + } + a->length = capacity; + return (JAVA_OBJECT)a; +} + +/* + * Read straight into this thread's buffer and hand back an array whose length is + * exactly the byte count, so the parser can scan to array.length as it always has. + * + * Setting `length` per read is the whole trick, and it is sound here for a reason + * worth stating: `length` is an ordinary int in a struct this file allocated and + * owns, the array is reachable only through the return value for the duration of + * one callback, and no Java reference points INTO the storage -- only at the + * header. A VM that packed the length into an object header the collector reads, + * or that moved objects, could not do this. + * + * Returns null at end of stream or on error, which the caller treats as the peer + * having gone away -- the same contract the copying path has. A ZERO-LENGTH array + * is the third answer: the descriptor had nothing ready. read() cannot produce it + * otherwise, since a zero-byte read IS end of stream, so the caller can tell the + * two apart. + */ +JAVA_OBJECT com_codename1_backend_ServerSocket_readIntoThreadBufferImpl___int_int_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_INT capacity) { + struct JavaArrayPrototype* a = cn1BackendEnsureReadArray(capacity); + ssize_t n; + int readErrno; + if(a == 0 || fd < 0) { + return JAVA_NULL; + } + // YIELD around the blocking read, exactly as readImpl does. Without it the + // thread stays marked active while it sits in the kernel, so the collector has + // to wait for every worker that is parked on a socket before it can stop the + // world. Omitting it cost HALF the throughput -- 147k against 288k req/s -- and + // it is a liveness bug before it is a performance one: a quiet connection + // would hold the collector for as long as the client stayed silent. + CN1_YIELD_THREAD; + do { + n = read(fd, cn1BackendReadStorage, (size_t)capacity); + } while(n < 0 && errno == EINTR); + readErrno = errno; + CN1_RESUME_THREAD; + if(n < 0 && (readErrno == EAGAIN || readErrno == EWOULDBLOCK)) { + // NOT end of stream. serveOne leaves plaintext descriptors non-blocking in + // virtual-thread mode, so a request whose bytes have not landed yet -- the + // headers in one packet and the first chunk in the next -- lands here, and + // reporting null dropped a perfectly good upload as though the peer had + // hung up. + // + // Answered rather than parked. readImpl parks on EAGAIN, but it reads into + // the CALLER'S array; this reads into a buffer that is __thread, so it is + // shared by every virtual thread multiplexed onto this host. Parking here + // hands the host to one of them, and its read would overwrite the storage + // this one is about to return -- trading a dropped upload for one request's + // bytes appearing inside another's. The caller falls back to the copying + // path instead, which parks correctly and owns its buffer. + struct JavaArrayPrototype* pending = cn1BackendEnsureWouldBlockArray(); + return pending == 0 ? JAVA_NULL : (JAVA_OBJECT)pending; + } + if(n <= 0) { + return JAVA_NULL; + } + a->length = (int)n; + return (JAVA_OBJECT)a; +} + +JAVA_INT com_codename1_backend_ServerSocket_boundPortImpl___int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd) { +#ifdef _WIN32 + (void)fd; + return -1; +#else + /* sockaddr_storage, because the bind above may have chosen IPv6 and the port + does not sit at the same offset in the two families -- reading a v6 socket + through sockaddr_in reports a number that was never the port. */ + struct sockaddr_storage addr; + socklen_t len = sizeof(addr); + if(fd < 0 || getsockname(fd, (struct sockaddr*)&addr, &len) != 0) { + return -1; + } + if(addr.ss_family == AF_INET6) { + return (JAVA_INT)ntohs(((struct sockaddr_in6*)&addr)->sin6_port); + } + return (JAVA_INT)ntohs(((struct sockaddr_in*)&addr)->sin_port); +#endif +} + +/* -1 means "nothing waiting" (EAGAIN) as well as a real error; the caller is a + poller that will be told again if there is more. */ +JAVA_INT com_codename1_backend_ServerSocket_acceptImpl___int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT serverFd) { +#ifdef _WIN32 + (void)serverFd; + return -1; +#else + int fd; + if(serverFd < 0) { + return -1; + } + CN1_YIELD_THREAD; + fd = accept(serverFd, NULL, NULL); + CN1_RESUME_THREAD; + if(fd < 0) { + return -1; + } + /* Nagle batches small writes, which on a request/response protocol means the + response header waits for the body. Off. */ + { + int on = 1; + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (const char*)&on, sizeof(on)); + } + return fd; +#endif +} + +JAVA_INT com_codename1_backend_ServerSocket_setBlockingImpl___int_boolean_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_BOOLEAN blocking) { +#ifdef _WIN32 + (void)fd; (void)blocking; + return -1; +#else + int flags; + if(fd < 0) { + return -1; + } + flags = fcntl(fd, F_GETFL, 0); + if(flags < 0) { + return -1; + } + flags = blocking ? (flags & ~O_NONBLOCK) : (flags | O_NONBLOCK); + return fcntl(fd, F_SETFL, flags) == 0 ? 0 : -1; +#endif +} + +/* + * A receive and send deadline for one descriptor, in milliseconds. + * + * This is what stops a connection that opens and then says nothing from holding a + * worker forever. The worker pool is bounded on purpose, so without a deadline a + * handful of silent connections is a complete denial of service -- open as many as + * there are workers and the server stops answering anyone. + */ +/* + * Wait for the socket to become readable, for at most timeoutMillis. + * + * ONE syscall, and it changes no socket state -- which is the whole point. The + * caller uses this between requests on a keep-alive connection, and the obvious + * alternatives both cost more: SO_RCVTIMEO has to be set and restored around + * every wait (two setsockopt each way, measured at 4 per request), and switching + * the descriptor to non-blocking costs an fcntl pair. poll leaves the descriptor + * exactly as it was, so the deadline that governs a real request read is never + * disturbed. + * + * Returns 1 readable, 0 timed out, -1 error. EINTR retries rather than reporting + * a timeout: the collector signals threads, and a signal is not a quiet client. + */ +JAVA_INT com_codename1_backend_ServerSocket_awaitReadableImpl___int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_INT timeoutMillis) { +#ifdef _WIN32 + (void)fd; (void)timeoutMillis; + return -1; +#else + struct pollfd p; + int rc; + if(fd < 0) { + return -1; + } + p.fd = fd; + p.events = POLLIN; + // On a virtual thread, ask once without blocking; if nothing is there, park + // rather than hold the host thread for the timeout. The scheduler only + // resumes a parked virtual thread once the poller reports its descriptor + // ready, so coming back IS the readiness answer. + if(cn1VirtualThreadCurrent() != 0) { + // ASK FIRST, and this poll is an optimisation rather than the waste it + // looks like in a syscall census. + // + // It was removed once on the grounds that Go does not do it -- its + // FD.Read calls read() straight away and parks on EAGAIN -- and that the + // census showed 1.9 ppoll per request against Go's zero. Throughput fell + // from 1047890 requests to about 110000 and /json died. The census was + // counting a cheap syscall that PREVENTS an expensive one: when the next + // request has already arrived, which under keep-alive it usually has, + // this answers immediately and the virtual thread never parks. Without + // it every keep-alive wait costs a park, an epoll round trip and a + // resume. + // + // The lesson generalises: syscall COUNT is not cost. Go can afford to + // skip this because its park is a goroutine switch inside a scheduler + // that is already awake; ours goes out to the poller and back. + // Left switchable rather than deleted, because the case against it is + // real and the case for it was measured on an older scheduler. + // + // Against: the caller's next move is a recv, and on a virtual thread that + // recv already parks on EAGAIN and retries when the poller says the + // descriptor is ready. Polling here first costs one extra syscall on + // EVERY request to learn what the read is about to learn anyway -- a + // corrected census puts it at exactly 1.0 ppoll per request, the only + // syscall in our profile that Go does not make at all. + // + // For: removing it costs almost all of the throughput, and that is still + // true after the host loop learned to drain a local run queue before + // polling -- which was the reason to expect otherwise. RE-MEASURED, four + // arms interleaved on a quiet host, /plaintext at 64 connections, medians + // of four steady-state reps: + // + // go 243,161 req/s + // virtual threads, probe 207,808 0.854 of go + // virtual threads, NO probe 20,134 0.082 of go <-- 12x worse + // + // The syscall census makes the trap explicit: without the probe a request + // costs 2.05 syscalls against Go's 2.64 -- FEWER than Go -- and it is ten + // times slower, because the ppoll it saves is replaced by a park, and a + // park is a poller round trip plus a resume. Syscall COUNT is not cost. + // Do not re-run this experiment expecting a different answer; run it only + // after the PARK itself gets cheaper. + // + // A zero timeout is a genuine probe (a caller asking "is anything there" + // without wanting to wait), so that one still has to ask regardless. + if(!cn1BackendSpeculativePoll() && timeoutMillis != 0) { + return 1; + } + p.revents = 0; + rc = poll(&p, 1, 0); + if(rc > 0) { + return 1; + } + if(rc < 0 && errno != EINTR) { + return -1; + } + if(timeoutMillis == 0) { + return 0; // a pure probe: no data, do not park + } + // YIELD around the park, and this is not optional bookkeeping. + // + // A parked virtual thread is not running and never will be until somebody + // resumes it, so leaving it marked ACTIVE tells the collector to wait for + // it to reach a safepoint that it cannot reach. This is the keep-alive + // wait, so in this mode every idle connection is parked here: the first + // burst of traffic works, and the moment the connections go quiet the + // collector stops being able to finish a cycle and every mutator ends up + // in the pacing park behind it. Observed exactly that -- 204712 requests + // served, then nothing, with no thread in epoll_pwait, five in futex, + // five in nanosleep, and the process at 7% CPU. + CN1_YIELD_THREAD; + cn1VirtualThreadYield(); + CN1_RESUME_THREAD; + return 1; + } + for(;;) { + int pollErrno; + p.revents = 0; + CN1_YIELD_THREAD; + rc = poll(&p, 1, timeoutMillis); + /* CAPTURED HERE, before CN1_RESUME_THREAD. The resume is a GC safepoint: it + can park this thread on a timed condition wait, and that leaves errno set + to ETIMEDOUT. Reading errno after it therefore reported the WAIT's outcome + rather than the poll's, so an ordinary EINTR from the collector's stop + signal was misread as a fatal poll error and a healthy keep-alive + connection was closed -- which the client sees as a reset, because there + is already a pipelined request sitting unread in the receive buffer. + Rare, load dependent, and it took several connections down at once because + one collector pause signals every worker. */ + pollErrno = errno; + CN1_RESUME_THREAD; + if(rc >= 0) { + break; + } + if(pollErrno != EINTR) { + return -1; + } + } + return rc > 0 ? 1 : 0; +#endif +} + +JAVA_INT com_codename1_backend_ServerSocket_setTimeoutImpl___int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_INT millis) { +#ifdef _WIN32 + (void)fd; (void)millis; + return -1; +#else + struct timeval tv; + if(fd < 0) { + return -1; + } + tv.tv_sec = millis / 1000; + tv.tv_usec = (millis % 1000) * 1000; + if(setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv)) != 0) { + return -1; + } + /* A send deadline too: a peer that stops reading would otherwise block a + worker in send() just as effectively as one that stops writing. */ + return setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (const char*)&tv, sizeof(tv)) == 0 ? 0 : -1; +#endif +} + +/* Blocking read while a worker owns the connection. -1 is end of stream, -2 an + error; the fd is set blocking before a worker gets it, so there is no EAGAIN. */ +JAVA_INT com_codename1_backend_ServerSocket_readImpl___int_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { +#ifdef _WIN32 + (void)fd; (void)buffer; (void)offset; (void)length; + return -2; +#else + JAVA_ARRAY_BYTE* data; + long n; + int readErrno; + if(fd < 0 || buffer == JAVA_NULL) { + return -2; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + CN1_YIELD_THREAD; + for(;;) { + n = (long)recv(fd, (char*)&data[offset], (size_t)length, 0); + if(n >= 0 || (errno != EINTR && errno != EAGAIN && errno != EWOULDBLOCK)) { + break; + } + if(errno == EINTR) { + continue; + } + // EAGAIN on a VIRTUAL thread is not an error and not a deadline: it means + // the bytes have not arrived. Park, and the scheduler resumes this virtual + // thread when the poller says the descriptor is readable -- the host thread + // goes and runs somebody else in the meantime, which is the entire point. + // + // On a platform thread there is no one to hand the host to, so the old + // answer stands: report the deadline and let the caller decide. + if(cn1VirtualThreadCurrent() == 0) { + break; + } + // The array may MOVE while we are parked -- a collection can run, and the + // buffer is an ordinary Java object -- so re-read the data pointer after + // every resume rather than trusting the one taken before the park. + cn1VirtualThreadYield(); + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + } + /* Captured before CN1_RESUME_THREAD for the same reason as the poll loop above: + the resume is a GC safepoint and can park this thread on a timed wait, which + overwrites errno. Reading it afterwards classified an ordinary deadline as a + fault (-2 instead of -3), and could equally hide a real error behind -3. */ + readErrno = errno; + CN1_RESUME_THREAD; + if(n == 0) { + return -1; + } + if(n < 0) { + /* -3 is the deadline expiring, which is an ordinary event a server sheds + rather than a fault worth logging as one. */ + return (readErrno == EAGAIN || readErrno == EWOULDBLOCK) ? -3 : -2; + } + return (JAVA_INT)n; +#endif +} + +/* + * Waits for a descriptor to become writable, bounded by its own send deadline. + * + * Needed because serveOne leaves plaintext descriptors NON-BLOCKING in + * virtual-thread mode, so send() answers EAGAIN as soon as the client's receive + * window fills. Both write paths treated that as a permanent failure and dropped + * the connection, which truncates any response a client reads slowly -- while a + * blocking descriptor, which is what they were written against, simply waited. + * + * Waiting here rather than parking the virtual thread: a park is resumed by the + * poller, and a connection descriptor is registered for READ only, so a thread + * parked on writability would never be woken. Yielding as RUNNABLE instead would + * spin, and a running thread has no idle deadline to expire it. SO_SNDTIMEO is + * already set per connection, so this bounds the wait the same way a blocking + * send would have. + * + * Returns 1 when writable, 0 on timeout, -1 on error. + */ +static int cn1AwaitWritable(int fd) { + struct pollfd waiting; + struct timeval tv; + socklen_t len = (socklen_t)sizeof(tv); + int timeout = -1; + int rc; + if(getsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (char*)&tv, &len) == 0) { + long millis = (long)tv.tv_sec * 1000L + (long)(tv.tv_usec / 1000); + if(millis > 0) { + timeout = (int)millis; + } + } + waiting.fd = fd; + waiting.events = POLLOUT; + waiting.revents = 0; + do { + rc = poll(&waiting, 1, timeout); + } while(rc < 0 && errno == EINTR); + if(rc < 0) { + return -1; + } + return rc == 0 ? 0 : 1; +} + +JAVA_INT com_codename1_backend_ServerSocket_writeImpl___int_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { +#ifdef _WIN32 + (void)fd; (void)buffer; (void)offset; (void)length; + return -1; +#else + JAVA_ARRAY_BYTE* data; + JAVA_INT written = 0; + if(fd < 0 || buffer == JAVA_NULL) { + return -1; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + CN1_YIELD_THREAD; + while(written < length) { + /* MSG_NOSIGNAL where it exists, so this write cannot raise SIGPIPE even if + the disposition were somehow restored. It is 0 on platforms without it -- + macOS among them -- where the SIG_IGN set at bind is what covers this. */ + long n = (long)send(fd, (const char*)&data[offset + written], + (size_t)(length - written), CN1_SEND_FLAGS); + if(n < 0 && errno == EINTR) { + continue; + } + if(n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + /* Backpressure, not failure: the client has not drained its window + yet. Returning -1 here dropped the connection and truncated the + response the moment a client read slower than the server wrote. */ + int ready = cn1AwaitWritable(fd); + if(ready > 0) { + continue; + } + CN1_RESUME_THREAD; + return ready == 0 ? -3 : -1; /* -3 is the deadline, as on the read side */ + } + if(n <= 0) { + CN1_RESUME_THREAD; + return -1; + } + written += (JAVA_INT)n; + } + CN1_RESUME_THREAD; + return written; +#endif +} + +JAVA_VOID com_codename1_backend_ServerSocket_closeFdImpl___int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd) { +#ifndef _WIN32 + if(fd >= 0) { + close(fd); + } +#else + (void)fd; +#endif +} + +/* ------------------------------------------------------------------ */ +/* Reactor */ +/* ------------------------------------------------------------------ */ + +JAVA_INT com_codename1_backend_Reactor_createImpl___R_int(CODENAME_ONE_THREAD_STATE) { +#if defined(CN1_HAVE_EPOLL) + return epoll_create1(0); +#elif defined(CN1_HAVE_KQUEUE) + return kqueue(); +#else + return -1; +#endif +} + +JAVA_INT com_codename1_backend_Reactor_registerImpl___int_int_int_boolean_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT poller, JAVA_INT fd, JAVA_INT events, JAVA_BOOLEAN modify) { +#if defined(CN1_HAVE_EPOLL) + struct epoll_event ev; + memset(&ev, 0, sizeof(ev)); + ev.data.fd = fd; + if(events & CN1_EVENT_READ) { + ev.events |= EPOLLIN; + } + if(events & CN1_EVENT_WRITE) { + ev.events |= EPOLLOUT; + } + if(events & CN1_EVENT_ONESHOT) { + // EPOLLONESHOT is what makes it safe for the WORKERS to poll the same + // epoll set directly instead of a reactor thread dispatching to them. + // After an event is delivered the kernel disarms the fd, so exactly one + // waiter can ever receive it and the "two workers on one connection" + // hazard that forces the reactor path to EPOLL_CTL_DEL before handing + // over cannot arise. Re-arming afterwards is one EPOLL_CTL_MOD, against + // the DEL + ADD that path pays, and it costs no cross-thread wake. + ev.events |= EPOLLONESHOT; + } + return epoll_ctl(poller, modify ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, fd, &ev) == 0 ? 0 : -1; +#elif defined(CN1_HAVE_KQUEUE) + struct kevent ev[2]; + int n = 0; + (void)modify; /* kevent's ADD is idempotent, so a modify is the same call */ + if(events & CN1_EVENT_READ) { + // EV_DISPATCH is kqueue's EPOLLONESHOT: deliver once, then disable the + // filter until it is re-enabled. EV_ENABLE on the re-arm turns it back on. + EV_SET(&ev[n++], fd, EVFILT_READ, + EV_ADD | EV_ENABLE | ((events & CN1_EVENT_ONESHOT) ? EV_DISPATCH : 0), + 0, 0, NULL); + } + if(events & CN1_EVENT_WRITE) { + EV_SET(&ev[n++], fd, EVFILT_WRITE, EV_ADD | EV_ENABLE, 0, 0, NULL); + } + if(n == 0) { + return 0; + } + return kevent(poller, ev, n, NULL, 0, NULL) < 0 ? -1 : 0; +#else + (void)poller; (void)fd; (void)events; (void)modify; + return -1; +#endif +} + +JAVA_INT com_codename1_backend_Reactor_unregisterImpl___int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT poller, JAVA_INT fd) { +#if defined(CN1_HAVE_EPOLL) + return epoll_ctl(poller, EPOLL_CTL_DEL, fd, NULL) == 0 ? 0 : -1; +#elif defined(CN1_HAVE_KQUEUE) + struct kevent ev[2]; + EV_SET(&ev[0], fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); + EV_SET(&ev[1], fd, EVFILT_WRITE, EV_DELETE, 0, 0, NULL); + /* ENOENT here just means it was not registered for that filter. */ + kevent(poller, ev, 2, NULL, 0, NULL); + return 0; +#else + (void)poller; (void)fd; + return -1; +#endif +} + +/* + * Fills readyFds with the descriptors that became ready and returns how many. + * Bracketed with CN1_YIELD_THREAD because this blocks for as long as the server + * is idle, which is most of its life -- without it the collector could not mark + * past the reactor thread. + */ +JAVA_INT com_codename1_backend_Reactor_waitImpl___int_int_1ARRAY_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_INT poller, JAVA_OBJECT readyFds, JAVA_INT timeoutMillis) { + JAVA_ARRAY arr; + JAVA_ARRAY_INT* out; + int capacity; + int count = 0; + if(readyFds == JAVA_NULL) { + return -1; + } + arr = (JAVA_ARRAY)readyFds; + out = (JAVA_ARRAY_INT*)arr->data; + capacity = arr->length; +#if defined(CN1_HAVE_EPOLL) + { + struct epoll_event* events = (struct epoll_event*)cn1BackendEnsureEventBuf( + capacity, sizeof(struct epoll_event)); + int n, i; + if(events == NULL) { + return -1; + } + CN1_YIELD_THREAD; + do { + n = epoll_wait(poller, events, capacity, timeoutMillis); + } while(n < 0 && errno == EINTR); + CN1_RESUME_THREAD; + for(i = 0 ; i < n && count < capacity ; i++) { + out[count++] = events[i].data.fd; + } + return n < 0 ? -1 : count; + } +#elif defined(CN1_HAVE_KQUEUE) + { + struct kevent* events = (struct kevent*)cn1BackendEnsureEventBuf( + capacity, sizeof(struct kevent)); + struct timespec ts; + struct timespec* tsp = NULL; + int n, i; + if(events == NULL) { + return -1; + } + if(timeoutMillis >= 0) { + ts.tv_sec = timeoutMillis / 1000; + ts.tv_nsec = (long)(timeoutMillis % 1000) * 1000000L; + tsp = &ts; + } + CN1_YIELD_THREAD; + do { + n = kevent(poller, NULL, 0, events, capacity, tsp); + } while(n < 0 && errno == EINTR); + CN1_RESUME_THREAD; + for(i = 0 ; i < n && count < capacity ; i++) { + out[count++] = (JAVA_INT)events[i].ident; + } + return n < 0 ? -1 : count; + } +#else + (void)poller; (void)timeoutMillis; + return -1; +#endif +} + +/* ===================== VIRTUAL THREADS FOR CONNECTIONS ===================== + * + * One virtual thread per connection, which is the shape Go gets from a goroutine + * per connection and the shape a bounded pool of OS threads cannot reach. The + * measured reason: handing a request between OS threads costs 21181ns here and + * switching a virtual thread costs 2.6ns. + * + * The scheduler is deliberately tiny, because the interesting part is done by + * the parking above. A host thread polls, resumes the virtual thread belonging + * to whichever descriptor is ready, and gets control back when that virtual + * thread either finishes the connection or parks waiting for more bytes. It + * never needs to know WHICH of those happened for any reason other than deciding + * whether to re-arm the descriptor. + */ + +/* The Java entry point a connection's virtual thread runs. Referencing the + * symbol here is also what keeps it alive: the dead-code pass treats a method + * named in native sources as used, and nothing in Java calls this one. */ +extern JAVA_VOID com_codename1_backend_HttpServer_serveVirtual___int(CODENAME_ONE_THREAD_STATE, JAVA_INT fd); + +struct cn1BackendVtArg { + JAVA_INT fd; +}; + +extern void markDeadThread(struct ThreadLocalData* d); + +static void cn1BackendVtBody(void* arg) { + struct cn1BackendVtArg* a = (struct cn1BackendVtArg*)arg; + /* getThreadLocalData returns THIS virtual thread's state, because it is the + * one running -- see the hook in nativeMethods.m. Taking the host's state + * here would give two threads of control one Java stack. */ + struct ThreadLocalData* mine = getThreadLocalData(); + com_codename1_backend_HttpServer_serveVirtual___int(mine, a->fd); + + /* + * RETIRE THE THREAD STATE. This is not tidiness, it is the difference between + * a server that works and one that stops after its first burst of traffic. + * + * A virtual thread's ThreadLocalData is registered in allThreads and marked + * lightweightThread, and the collector's stop-the-world does this for every + * such entry: + * + * t->threadBlockedByGC = JAVA_TRUE; + * while(t->threadActive) { usleep(500); } // no timeout + * + * A finished virtual thread will never run again, so nothing will ever clear + * threadActive for it, and the collector waits on it for ever. Every GC cycle + * after the first connection closes simply never completes; the allocation + * pacing then never releases, and the whole server settles to a few hundred + * requests a second while looking completely idle -- no crash, no spin, 8% of + * a CPU. Found by asking gdb where the collector was, and reading what it was + * waiting for. + * + * A platform thread has exactly this call at the end of threadRunner, for + * exactly this reason. A virtual thread needs it just as much: it is a Java + * thread of control as far as the collector is concerned, and it has to + * announce its own death. + */ + /* + * Say "not running" here, but do NOT retire the state here. + * + * The collector waits on threadActive for any lightweightThread, so clearing + * it closes the window between this virtual thread finishing and its host + * getting round to freeing it. Retiring the state is a different matter: + * markDeadThread calls collectThreadResources, which frees threadObjectStack + * -- the Java stack this function is still standing on. Doing it here killed + * the process inside the first burst. It belongs on the host, after the + * switch back, which is where freeImpl runs. + */ + mine->threadActive = JAVA_FALSE; +} + +/* Instrumentation: a virtual thread that is never resumed again is still + * registered with the collector, and if it is marked active the collector waits + * for a safepoint it can never reach. Counting created against freed says + * whether that is happening without having to infer it from thread states. */ +static _Atomic long cn1VtCreated = 0; +static _Atomic long cn1VtFreed = 0; +static _Atomic long cn1VtFinished = 0; + +static void cn1VtReport(const char* why) { + fprintf(stderr, "[CN1-VT] %s created=%ld finished=%ld freed=%ld live=%ld\n", why, + atomic_load(&cn1VtCreated), atomic_load(&cn1VtFinished), + atomic_load(&cn1VtFreed), + atomic_load(&cn1VtCreated) - atomic_load(&cn1VtFreed)); +} + +JAVA_VOID com_codename1_backend_VirtualThread_reportImpl__(CODENAME_ONE_THREAD_STATE) { + cn1VtReport("report"); +} + +JAVA_LONG com_codename1_backend_VirtualThread_createImpl___int_int_R_long(CODENAME_ONE_THREAD_STATE, JAVA_INT fd, JAVA_INT stackBytes) { + struct cn1BackendVtArg* a; + struct cn1VirtualThread* vt; + a = (struct cn1BackendVtArg*)malloc(sizeof(struct cn1BackendVtArg)); + if(a == 0) { + return 0; + } + a->fd = fd; + vt = cn1SpawnVirtualThread(cn1BackendVtBody, a, (size_t)stackBytes); + if(vt == 0) { + static int reported = 0; + if(!reported) { + reported = 1; + fprintf(stderr, "[CN1-VT] spawn failed (stackBytes=%d, errno=%d %s)\n", + (int)stackBytes, errno, strerror(errno)); + } + free(a); + return 0; + } + atomic_fetch_add(&cn1VtCreated, 1); + return (JAVA_LONG)(intptr_t)vt; +} + +/* True once the connection is done with. False means it parked and is waiting + * for its descriptor to become readable again. */ +/* + * 0 finished, 1 parked waiting for its descriptor, 2 yielded but RUNNABLE. + * + * The third answer is the one that matters. A virtual thread that gave up its + * host inside the collector's allocation backpressure is not waiting for bytes: + * handing its descriptor to the poller waits for a client that is itself waiting + * for the response this virtual thread still owes it, and neither side ever + * moves. It has to go back on a run queue instead. + */ +JAVA_INT com_codename1_backend_VirtualThread_resumeImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + struct cn1VirtualThread* vt = (struct cn1VirtualThread*)(intptr_t)handle; + if(vt == 0) { + return 0; + } + cn1VirtualThreadSetYieldReason(CN1_VT_YIELD_IO); /* the default for a plain park */ + cn1VirtualThreadResume(vt); + if(cn1VirtualThreadFinished(vt)) { + atomic_fetch_add(&cn1VtFinished, 1); + return 0; + } + return cn1VirtualThreadYieldReason(vt) == CN1_VT_YIELD_RUNNABLE ? 2 : 1; +} + +/* The descriptor this virtual thread serves. The run queue holds handles, and a + * handle that comes back from the queue has to be matched to its slot again. */ +JAVA_INT com_codename1_backend_VirtualThread_descriptorImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + struct cn1VirtualThread* vt = (struct cn1VirtualThread*)(intptr_t)handle; + struct cn1BackendVtArg* a; + if(vt == 0) { + return -1; + } + a = (struct cn1BackendVtArg*)cn1VirtualThreadArg(vt); + return a == 0 ? -1 : a->fd; +} + +JAVA_VOID com_codename1_backend_VirtualThread_freeImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + struct cn1VirtualThread* vt = (struct cn1VirtualThread*)(intptr_t)handle; + struct ThreadLocalData* victim; + if(vt == 0) { + return; + } + /* + * RETIRE THE VIRTUAL THREAD'S VM STATE, and this is the whole bug fixed. + * + * That state is registered in allThreads and flagged lightweightThread, and + * the collector's stop-the-world does this for every such entry: + * + * t->threadBlockedByGC = JAVA_TRUE; + * while(t->threadActive) { usleep(500); } // no timeout + * + * A finished virtual thread never runs again, so if its state stays in that + * list the collector waits on it for ever: every cycle after the first + * connection closes fails to complete, the allocation pacing never releases, + * and the server settles at a few hundred requests a second while looking + * completely idle -- 8% of a CPU, no crash, no spin. A platform thread makes + * exactly this call at the end of threadRunner; a virtual thread is a Java + * thread of control to the collector and owes it the same announcement. + * + * Here rather than at the end of the body because markDeadThread frees the + * thread's object stack, and the body is still standing on it. + */ + victim = (struct ThreadLocalData*)cn1VirtualThreadState(vt); + if(victim != 0) { + cn1VirtualThreadSetState(vt, 0); + markDeadThread(victim); + /* + * ASK FOR THE RELEASE. markDeadThread only QUEUES the state: it sets + * gcQueuedForDrain and hands the TLD to cn1DrainDeadThreadPending, which + * migrates the pending allocations and then frees the TLD only if + * gcReleaseRequested is set. Nothing else sets it for a virtual thread -- + * an OS thread gets it from the Thread object's finalizer, and + * cn1RetireVirtualThread (the VM's own retirement path, which this native + * duplicates) sets it right here for exactly this reason. + * + * Without it the drain runs, clears gcQueuedForDrain, and walks away + * leaving the TLD allocated for ever. That is ~68KB per connection -- + * callStack arrays ~50KB, pendingHeapAllocations ~27KB, the try-block + * array ~15KB, all malloc'd -- and it never comes back: 900 closed + * connections took resident memory from 3MB to 65MB, and 249 collections + * returned none of it, because every one of those drains found the flag + * clear. + * + * Deferred rather than freed here, and that is deliberate: codenameOneGCMark + * copies each ThreadLocalData* out of allThreads under the critical section + * and dereferences it outside, so a mark already past that copy still holds + * this pointer. The drain runs at the start of the next mark, which is the + * one point where no collector iteration can. + */ + lockCriticalSection(); + victim->gcReleaseRequested = JAVA_TRUE; + unlockCriticalSection(); + } + /* The argument block outlives the body, so it is freed here rather than at + * the end of the body: the body's stack frame is gone by then. */ + free(cn1VirtualThreadArg(vt)); + cn1VirtualThreadFree(vt); + atomic_fetch_add(&cn1VtFreed, 1); +} + +/* + * Whether this build actually has the context switch, so the server can DEFAULT + * to virtual threads without breaking a target that lacks them. + * + * cn1_virtual_thread.h compiles the real implementation only on non-Windows + * aarch64/x86_64; everywhere else every entry point is a stub and + * cn1SpawnVirtualThread returns 0. A default of "virtual threads" that did not + * ask this would drop every connection on those targets rather than fall back. + */ +JAVA_BOOLEAN com_codename1_backend_VirtualThread_supportedImpl___R_boolean(CODENAME_ONE_THREAD_STATE) { +#ifdef CN1_VIRTUAL_THREADS + return JAVA_TRUE; +#else + return JAVA_FALSE; +#endif +} + +JAVA_BOOLEAN com_codename1_backend_VirtualThread_isVirtualImpl___R_boolean(CODENAME_ONE_THREAD_STATE) { + return cn1VirtualThreadCurrent() != 0 ? JAVA_TRUE : JAVA_FALSE; +} + +/* + * Give up the host thread without waiting for anything. + * + * A virtual thread parks by itself when the bytes it wants have not arrived, and + * under a load generator that always has the next request queued that never + * happens -- so a virtual thread would hold its host for as long as the client + * kept talking, and with fewer hosts than connections the rest starve. Removing + * the burst cap entirely produced exactly that: two hosts serving two of sixty + * four connections. The cap has to stay; what was wrong was closing the + * connection to honour it rather than stepping aside. + */ +JAVA_VOID com_codename1_backend_VirtualThread_yieldImpl__(CODENAME_ONE_THREAD_STATE) { + if(cn1VirtualThreadCurrent() != 0) { + // Marked inactive across the switch for the same reason the keep-alive + // park is: a virtual thread that is not on a host cannot answer the + // collector, and a collector waiting for it stops the whole server. + CN1_YIELD_THREAD; + cn1VirtualThreadYield(); + CN1_RESUME_THREAD; + } +} + +/* + * Cores available to this process. + * + * Virtual-thread mode needs it because the host count must track the cores and + * not the expected concurrency: concurrency comes from the virtual threads, so + * a host per core is enough, and more than that is actively harmful. Measured on + * two pinned cores, 16 hosts served 117 requests where 2 served 257297 -- the + * host threads simply contend for the cores the server needs. Unpinned, where + * the machine has cores to spare, every host count from 2 to 32 behaves and the + * difference disappears, which is why this has to be read at runtime rather than + * guessed at build time. + */ +JAVA_INT com_codename1_backend_ServerSocket_availableProcessorsImpl___R_int(CODENAME_ONE_THREAD_STATE) { +#ifdef _WIN32 + return 1; +#else + long n = sysconf(_SC_NPROCESSORS_ONLN); + return n > 0 ? (JAVA_INT)n : 1; +#endif +} diff --git a/vm/backend/native/cn1_backend_signals.c b/vm/backend/native/cn1_backend_signals.c new file mode 100644 index 00000000000..1e9d0dd0140 --- /dev/null +++ b/vm/backend/native/cn1_backend_signals.c @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Waiting for SIGTERM, the way a container tells a process to stop. + * + * The self-pipe trick: the signal handler does one write() of a single byte -- one + * of the few calls that is async-signal-safe -- and a Java thread turns the + * asynchronous event into an ordinary blocking read. + * + * The obvious alternative, blocking the signals everywhere and calling sigwait() + * on a dedicated thread, does NOT work here. pthread_sigmask only affects the + * calling thread and threads created after it, and ParparVM has already started + * its collector thread before main() runs. A signal delivered to that thread finds + * it unblocked and takes the default action, which is to kill the process -- + * measured: SIGTERM terminated the server with an in-flight request still open and + * no shutdown hook ever ran. + * + * A handler that called into the VM instead would be worse: it runs on whichever + * thread the signal lands on, in async-signal-safe context, so allocating, taking + * a monitor or touching the collector from it is undefined. + */ +#include "cn1_globals.h" +#include +#include +#include +#ifndef _WIN32 +#include +#include +#include +#endif + +#ifndef _WIN32 +static int cn1SignalPipe[2] = {-1, -1}; + +static void cn1SignalHandler(int signo) { + unsigned char byte = (unsigned char)signo; + /* write() is async-signal-safe; nothing else here would be. The result is + deliberately ignored: a full pipe means a shutdown is already pending. */ + ssize_t ignored = write(cn1SignalPipe[1], &byte, 1); + (void)ignored; +} +#endif + +JAVA_INT com_codename1_backend_Signals_blockImpl___R_int(CODENAME_ONE_THREAD_STATE) { +#ifdef _WIN32 + return -1; +#else + struct sigaction sa; + if(cn1SignalPipe[0] >= 0) { + return 0; /* already installed */ + } + if(pipe(cn1SignalPipe) != 0) { + return -1; + } + /* The write end must not block inside the handler. */ + fcntl(cn1SignalPipe[1], F_SETFL, fcntl(cn1SignalPipe[1], F_GETFL, 0) | O_NONBLOCK); + fcntl(cn1SignalPipe[0], F_SETFD, FD_CLOEXEC); + fcntl(cn1SignalPipe[1], F_SETFD, FD_CLOEXEC); + + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = cn1SignalHandler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; /* do not turn every blocking call into EINTR */ + if(sigaction(SIGINT, &sa, NULL) != 0 || sigaction(SIGTERM, &sa, NULL) != 0) { + return -1; + } + /* SIGPIPE is ignored rather than caught: writing to a socket whose peer has + gone is routine for a server, and the default action is to kill the process. + Ignored, the write returns EPIPE like any other error. */ + signal(SIGPIPE, SIG_IGN); + return 0; +#endif +} + +/* Blocks until SIGINT or SIGTERM arrives; returns the signal number, or -1. */ +JAVA_INT com_codename1_backend_Signals_awaitImpl___R_int(CODENAME_ONE_THREAD_STATE) { +#ifdef _WIN32 + return -1; +#else + unsigned char byte = 0; + ssize_t n; + if(cn1SignalPipe[0] < 0) { + return -1; + } + CN1_YIELD_THREAD; + do { + n = read(cn1SignalPipe[0], &byte, 1); + } while(n < 0 && errno == EINTR); + CN1_RESUME_THREAD; + return n == 1 ? (JAVA_INT)byte : -1; +#endif +} diff --git a/vm/backend/native/cn1_backend_tls.c b/vm/backend/native/cn1_backend_tls.c new file mode 100644 index 00000000000..d4d9fe39c67 --- /dev/null +++ b/vm/backend/native/cn1_backend_tls.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Server-side TLS on OpenSSL. + * + * One SSL_CTX for the process (it holds the certificate and the session cache) and + * one SSL per connection. The handshake runs on the worker that picks the + * connection up, where the descriptor is already blocking -- doing it on the + * reactor thread would block every other connection behind one slow client. + * + * TLS 1.2 is the floor. Everything below it is broken in ways that are not worth + * carrying, and OpenSSL's defaults above that are better than a hand-written + * cipher list that goes stale. + */ +#include "cn1_globals.h" +#include +#include +#include +#ifndef _WIN32 +#include /* CN1_RESUME_THREAD expands to usleep */ +#endif +#include +#include + +static int cn1TlsInitialised = 0; + +/* + * ALPN. HTTP/2 over TLS is only ever reached this way -- there is no upgrade + * handshake for h2 over TLS, so a server that does not advertise "h2" here will + * never speak it however complete the rest of its implementation is. + * + * The wire format is a list of length-prefixed names. h2 is offered first so a + * client that supports both gets it; http/1.1 stays in the list because most + * clients still ask for it and a server that only offers h2 refuses them. + */ +static const unsigned char CN1_ALPN_BOTH[] = { 2, 'h', '2', 8, 'h', 't', 't', 'p', '/', '1', '.', '1' }; +static const unsigned char CN1_ALPN_HTTP11[] = { 8, 'h', 't', 't', 'p', '/', '1', '.', '1' }; +/* The h2 policy travels in the callback's own arg rather than in a variable + beside it: a process that serves two TLS ports had the second createContext + overwrite the first one's setting, and every context shares this callback, so + a server built for http/1.1 could start negotiating h2 (or stop offering it) + because of an unrelated server elsewhere in the same process. */ + +static int cn1AlpnSelect(SSL* ssl, const unsigned char** out, unsigned char* outlen, + const unsigned char* in, unsigned int inlen, void* arg) { + int offerH2 = (int)(intptr_t)arg; + const unsigned char* offered = offerH2 ? CN1_ALPN_BOTH : CN1_ALPN_HTTP11; + unsigned int offeredLen = offerH2 ? (unsigned int)sizeof(CN1_ALPN_BOTH) + : (unsigned int)sizeof(CN1_ALPN_HTTP11); + (void)ssl; + if(SSL_select_next_proto((unsigned char**)out, outlen, offered, offeredLen, in, inlen) + != OPENSSL_NPN_NEGOTIATED) { + /* No overlap. NOACK rather than ALERT_FATAL: a client that offered only + protocols we do not speak still gets a working http/1.1 connection, + which is what it would have had with no ALPN at all. */ + return SSL_TLSEXT_ERR_NOACK; + } + return SSL_TLSEXT_ERR_OK; +} + +JAVA_LONG com_codename1_backend_Tls_createContextImpl___java_lang_String_java_lang_String_boolean_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT certPath, JAVA_OBJECT keyPath, JAVA_BOOLEAN offerHttp2) { + SSL_CTX* ctx; + char* cert; + const char* key; + if(certPath == JAVA_NULL || keyPath == JAVA_NULL) { + return 0; + } + if(!cn1TlsInitialised) { + SSL_library_init(); + SSL_load_error_strings(); + cn1TlsInitialised = 1; + } + /* stringToUTF8 hands back this thread's scratch buffer, so the first path is + copied before the second conversion overwrites it. */ + { + const char* tmp = stringToUTF8(threadStateData, certPath); + if(tmp == NULL) { + return 0; + } + cert = strdup(tmp); + } + key = stringToUTF8(threadStateData, keyPath); + if(key == NULL) { + free(cert); + return 0; + } + ctx = SSL_CTX_new(TLS_server_method()); + if(ctx == NULL) { + free(cert); + return 0; + } + SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION); + SSL_CTX_set_alpn_select_cb(ctx, cn1AlpnSelect, + (void*)(intptr_t)(offerHttp2 ? 1 : 0)); + /* The handshake and the record layer both want to retry on a partial write + with a moved buffer; without this OpenSSL refuses and the connection dies + on a large response. */ + SSL_CTX_set_mode(ctx, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER | SSL_MODE_AUTO_RETRY); + if(SSL_CTX_use_certificate_chain_file(ctx, cert) != 1 || + SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) != 1 || + SSL_CTX_check_private_key(ctx) != 1) { + SSL_CTX_free(ctx); + free(cert); + return 0; + } + free(cert); + return (JAVA_LONG)(intptr_t)ctx; +} + +JAVA_VOID com_codename1_backend_Tls_freeContextImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + SSL_CTX* ctx = (SSL_CTX*)(intptr_t)handle; + if(ctx != NULL) { + SSL_CTX_free(ctx); + } +} + +/* Runs the handshake. Returns the session handle, or 0. */ +JAVA_LONG com_codename1_backend_Tls_acceptImpl___long_int_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG ctxHandle, JAVA_INT fd) { + SSL_CTX* ctx = (SSL_CTX*)(intptr_t)ctxHandle; + SSL* ssl; + int rc; + if(ctx == NULL || fd < 0) { + return 0; + } + ssl = SSL_new(ctx); + if(ssl == NULL) { + return 0; + } + if(SSL_set_fd(ssl, fd) != 1) { + SSL_free(ssl); + return 0; + } + CN1_YIELD_THREAD; + rc = SSL_accept(ssl); + CN1_RESUME_THREAD; + if(rc != 1) { + /* A failed handshake is ordinary traffic -- a scanner, a client with no + common cipher, a plaintext request to an https port. Drain the error + queue so it cannot be misattributed to the next connection on this + thread. */ + ERR_clear_error(); + SSL_free(ssl); + return 0; + } + return (JAVA_LONG)(intptr_t)ssl; +} + +/* -1 at end of stream, -2 on error, otherwise the byte count. */ +JAVA_INT com_codename1_backend_Tls_readImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + SSL* ssl = (SSL*)(intptr_t)handle; + JAVA_ARRAY_BYTE* data; + int n; + if(ssl == NULL || buffer == JAVA_NULL) { + return -2; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + CN1_YIELD_THREAD; + n = SSL_read(ssl, &data[offset], length); + CN1_RESUME_THREAD; + if(n > 0) { + return (JAVA_INT)n; + } + { + int err = SSL_get_error(ssl, n); + ERR_clear_error(); + if(err == SSL_ERROR_ZERO_RETURN) { + return -1; /* the peer closed the session cleanly */ + } + return -2; + } +} + +JAVA_INT com_codename1_backend_Tls_writeImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + SSL* ssl = (SSL*)(intptr_t)handle; + JAVA_ARRAY_BYTE* data; + JAVA_INT written = 0; + if(ssl == NULL || buffer == JAVA_NULL) { + return -1; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + CN1_YIELD_THREAD; + while(written < length) { + int n = SSL_write(ssl, &data[offset + written], length - written); + if(n <= 0) { + ERR_clear_error(); + CN1_RESUME_THREAD; + return -1; + } + written += (JAVA_INT)n; + } + CN1_RESUME_THREAD; + return written; +} + +/* The protocol ALPN settled on: "h2", "http/1.1", or null when there was none. */ +JAVA_OBJECT com_codename1_backend_Tls_negotiatedProtocolImpl___long_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + SSL* ssl = (SSL*)(intptr_t)handle; + const unsigned char* proto = NULL; + unsigned int len = 0; + char name[32]; + if(ssl == NULL) { + return JAVA_NULL; + } + SSL_get0_alpn_selected(ssl, &proto, &len); + if(proto == NULL || len == 0 || len >= sizeof(name)) { + return JAVA_NULL; + } + memcpy(name, proto, len); + name[len] = 0; + return newStringFromCString(threadStateData, name); +} + +JAVA_VOID com_codename1_backend_Tls_closeImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + SSL* ssl = (SSL*)(intptr_t)handle; + if(ssl == NULL) { + return; + } + /* One shutdown, not the two-step wait for the peer's close_notify: a client + that has already gone would otherwise hold the worker until the deadline. */ + SSL_shutdown(ssl); + ERR_clear_error(); + SSL_free(ssl); +} diff --git a/vm/backend/native/cn1_backend_tlsclient.c b/vm/backend/native/cn1_backend_tlsclient.c new file mode 100644 index 00000000000..77f28df062e --- /dev/null +++ b/vm/backend/native/cn1_backend_tlsclient.c @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Outbound (client-side) TLS on OpenSSL, as an UPGRADE of a connected socket. + * + * It is an upgrade rather than a secure connect because that is the shape the + * database protocols need: PostgreSQL sends an SSLRequest packet and MySQL an + * SSLRequest capability flag, both in plaintext, and only then does the handshake + * begin on the same descriptor. A connect-time flag could not express that. + * + * Two things here are the whole security value, and both are easy to leave out: + * + * - SSL_CTX_set_default_verify_paths plus SSL_VERIFY_PEER, so an untrusted chain + * fails the handshake. Without the mode, OpenSSL completes the handshake and + * reports the failure only if you go looking, which nobody does. + * - SSL_set1_host, so the certificate has to be FOR the host we asked for. A + * verified chain for someone else's name is not authentication, and OpenSSL + * does not check the name unless it is told to. + * + * SNI is sent separately (SSL_set_tlsext_host_name): it tells the server which + * certificate to present and proves nothing on its own. + * + * Built as a stub when the backend is compiled without TLS (CN1_BACKEND_NO_TLS), + * rather than left out of the build: Tcp always declares these natives, and a + * native whose symbol is absent is dropped from the Java side by the dead-code + * pass, which would make Tcp.startTls silently do nothing. + */ +#include "cn1_globals.h" +#include +#include +#include +#include + +#ifndef CN1_BACKEND_NO_TLS + +#ifndef _WIN32 +#include /* CN1_RESUME_THREAD expands to usleep */ +#include +#include +#include /* inet_pton, for telling an IP literal from a DNS name */ +#endif +#include +#include +#include + +static int cn1ClientTlsInitialised = 0; +/* One context per trust root, because a context holds the trust store. The + * system store is the common case and gets slot 0; a caller that supplies its own + * CA bundle -- which is how a managed database or a development container is + * reached -- gets a slot keyed by the file's path. The table is small and never + * shrinks: the number of distinct trust roots a process uses is the number of + * databases and services it talks to. */ +#define CN1_TLS_CONTEXT_SLOTS 8 +static SSL_CTX* cn1ClientTlsContexts[CN1_TLS_CONTEXT_SLOTS]; +static char cn1ClientTlsRoots[CN1_TLS_CONTEXT_SLOTS][1024]; +static int cn1ClientTlsContextCount = 0; +/* + * The cache is shared by every request thread, so building an entry has to be + * exclusive. Two threads opening their first TLS connection at once could pick the + * same slot and interleave the strcpy of the root name with the store of the + * context, leaving an entry labelled for one CA bundle holding the context built + * for another -- a later connection then validates against a trust root the caller + * did not choose, which is the one failure mode TLS exists to prevent. + */ +static pthread_mutex_t cn1ClientTlsMutex = PTHREAD_MUTEX_INITIALIZER; +/* The last handshake failure, for the message Java throws. Per process rather + * than per thread: a failed connect is reported immediately by the thread that + * saw it, and a race here would at worst attach the wrong reason to a failure + * that happened anyway. */ +static char cn1ClientTlsError[512]; + +static void cn1ClientTlsRecordError(const char* stage) { + unsigned long code = ERR_get_error(); + char buffer[256]; + buffer[0] = 0; + if(code != 0) { + ERR_error_string_n(code, buffer, sizeof(buffer)); + } + snprintf(cn1ClientTlsError, sizeof(cn1ClientTlsError), "%s%s%s", stage, + buffer[0] ? ": " : "", buffer); +} + +/* Holds cn1ClientTlsMutex for the whole lookup-and-build; see the mutex above. */ +static SSL_CTX* cn1ClientTlsEnsureContextLocked(const char* caFile) { + const char* key = caFile == 0 ? "" : caFile; + SSL_CTX* ctx; + int iter; + for(iter = 0 ; iter < cn1ClientTlsContextCount ; iter++) { + if(strcmp(cn1ClientTlsRoots[iter], key) == 0) { + return cn1ClientTlsContexts[iter]; + } + } + if(cn1ClientTlsContextCount >= CN1_TLS_CONTEXT_SLOTS) { + snprintf(cn1ClientTlsError, sizeof(cn1ClientTlsError), + "too many distinct TLS trust roots (limit %d)", CN1_TLS_CONTEXT_SLOTS); + return 0; + } + if(strlen(key) >= sizeof(cn1ClientTlsRoots[0])) { + snprintf(cn1ClientTlsError, sizeof(cn1ClientTlsError), "the CA path is too long"); + return 0; + } + if(!cn1ClientTlsInitialised) { + SSL_library_init(); + SSL_load_error_strings(); + cn1ClientTlsInitialised = 1; + } + ctx = SSL_CTX_new(TLS_client_method()); + if(ctx == 0) { + cn1ClientTlsRecordError("could not create a TLS context"); + return 0; + } + /* TLS 1.2 is the floor; everything below it is broken in ways not worth + * carrying, and OpenSSL's defaults above it beat a hand-written cipher list + * that goes stale. */ + SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION); + if(key[0] == 0) { + if(SSL_CTX_set_default_verify_paths(ctx) != 1) { + /* No system trust store. Refuse rather than fall back to trusting + * everything: an unverified connection that looks encrypted is worse + * than a plaintext one that looks plaintext. */ + cn1ClientTlsRecordError("no system CA store is available"); + SSL_CTX_free(ctx); + return 0; + } + } else if(SSL_CTX_load_verify_locations(ctx, key, 0) != 1) { + /* The caller named a CA bundle and it did not load. Falling back to the + * system store would verify against roots the caller deliberately did not + * choose, which is not what was asked for. */ + cn1ClientTlsRecordError("could not load the CA bundle"); + SSL_CTX_free(ctx); + return 0; + } + SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, 0); + strcpy(cn1ClientTlsRoots[cn1ClientTlsContextCount], key); + cn1ClientTlsContexts[cn1ClientTlsContextCount] = ctx; + /* The count LAST: a reader that sees it has already seen both writes above it. */ + cn1ClientTlsContextCount++; + return ctx; +} + +static SSL_CTX* cn1ClientTlsEnsureContext(const char* caFile) { + SSL_CTX* ctx; + pthread_mutex_lock(&cn1ClientTlsMutex); + ctx = cn1ClientTlsEnsureContextLocked(caFile); + pthread_mutex_unlock(&cn1ClientTlsMutex); + return ctx; +} + +/* + * Whether this host is an IP literal rather than a DNS name. + * + * inet_pton is the check, not a scan for dots and digits: "1.2.3.4.5" and + * "999.1.1.1" look like addresses to a hand-rolled test and are not ones, and a + * name wrongly treated as an address would be verified against IP SANs it can + * never have. v6 is tried as well, with the brackets a URL may carry removed. + */ +static int cn1IsIpLiteral(const char* host) { + struct in_addr v4; + struct in6_addr v6; + char trimmed[64]; + size_t length; + if(host == NULL) { + return 0; + } + if(inet_pton(AF_INET, host, &v4) == 1) { + return 1; + } + length = strlen(host); + if(length >= 2 && host[0] == '[' && host[length - 1] == ']') { + if(length - 2 >= sizeof(trimmed)) { + return 0; + } + memcpy(trimmed, host + 1, length - 2); + trimmed[length - 2] = 0; + return inet_pton(AF_INET6, trimmed, &v6) == 1; + } + return inet_pton(AF_INET6, host, &v6) == 1; +} + +JAVA_LONG com_codename1_backend_Tcp_startTlsImpl___long_java_lang_String_java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT host, JAVA_OBJECT caFile) { + SSL_CTX* ctx; + SSL* ssl; + char* h; + const char* ca; + int fd = handle <= 0 ? -1 : (int)(handle - 1); + int rc; + if(fd < 0) { + snprintf(cn1ClientTlsError, sizeof(cn1ClientTlsError), "the socket is closed"); + return 0; + } + /* stringToUTF8 hands back THIS THREAD'S single scratch buffer, and the next + * conversion frees and reallocates it. The host has to be copied before the + * CA path is converted; using it afterwards is a use-after-free that presents + * as a wild pointer somewhere else entirely. */ + { + const char* tmp = host == JAVA_NULL ? 0 : stringToUTF8(threadStateData, host); + if(tmp == 0) { + snprintf(cn1ClientTlsError, sizeof(cn1ClientTlsError), + "no host name to verify against"); + return 0; + } + h = strdup(tmp); + if(h == 0) { + snprintf(cn1ClientTlsError, sizeof(cn1ClientTlsError), "out of memory"); + return 0; + } + } + ca = caFile == JAVA_NULL ? 0 : stringToUTF8(threadStateData, caFile); + ctx = cn1ClientTlsEnsureContext(ca); + if(ctx == 0) { + free(h); + return 0; + } + ssl = SSL_new(ctx); + if(ssl == 0) { + cn1ClientTlsRecordError("could not create a TLS session"); + free(h); + return 0; + } + SSL_set_fd(ssl, fd); + SSL_set_tlsext_host_name(ssl, h); + /* The name check. Without it a valid certificate for any other host would + * pass, which is most of what TLS is for here. + * + * An IP literal takes a DIFFERENT call. SSL_set1_host matches DNS names and + * does not look at iPAddress subjectAltNames at all, so a database URL naming + * a host by address failed verification against a certificate that correctly + * carried the IP -- after packaging only, since the Java SE arm checks both. + * X509_VERIFY_PARAM_set1_ip_asc is the IP half of the same door. */ + if(cn1IsIpLiteral(h)) { + if(X509_VERIFY_PARAM_set1_ip_asc(SSL_get0_param(ssl), h) != 1) { + cn1ClientTlsRecordError("could not set the expected peer address"); + SSL_free(ssl); + free(h); + return 0; + } + } else if(SSL_set1_host(ssl, h) != 1) { + cn1ClientTlsRecordError("could not set the expected host name"); + SSL_free(ssl); + free(h); + return 0; + } + CN1_YIELD_THREAD; + rc = SSL_connect(ssl); + CN1_RESUME_THREAD; + free(h); + if(rc != 1) { + long verify = SSL_get_verify_result(ssl); + if(verify != X509_V_OK) { + snprintf(cn1ClientTlsError, sizeof(cn1ClientTlsError), + "certificate rejected: %s", X509_verify_cert_error_string(verify)); + } else { + cn1ClientTlsRecordError("handshake failed"); + } + SSL_free(ssl); + return 0; + } + return (JAVA_LONG)(intptr_t)ssl; +} + +JAVA_OBJECT com_codename1_backend_Tcp_tlsErrorImpl___R_java_lang_String(CODENAME_ONE_THREAD_STATE) { + return newStringFromCString(threadStateData, + cn1ClientTlsError[0] ? cn1ClientTlsError : "unknown TLS failure"); +} + +JAVA_INT com_codename1_backend_Tcp_tlsReadImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG session, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + SSL* ssl = (SSL*)(intptr_t)session; + JAVA_ARRAY_BYTE* data; + int n; + if(ssl == 0 || buffer == JAVA_NULL) { + return -2; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + CN1_YIELD_THREAD; + n = SSL_read(ssl, (char*)&data[offset], (int)length); + CN1_RESUME_THREAD; + if(n > 0) { + return (JAVA_INT)n; + } + /* A clean close_notify is end of stream, not an error; anything else is. */ + if(SSL_get_error(ssl, n) == SSL_ERROR_ZERO_RETURN) { + return -1; + } + return -2; +} + +JAVA_INT com_codename1_backend_Tcp_tlsWriteImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG session, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + SSL* ssl = (SSL*)(intptr_t)session; + JAVA_ARRAY_BYTE* data; + int written = 0; + if(ssl == 0 || buffer == JAVA_NULL) { + return -2; + } + data = (JAVA_ARRAY_BYTE*)((JAVA_ARRAY)buffer)->data; + /* SSL_write can return a short count, and the caller checks for the full + * length, so the loop is here rather than in Java. */ + while(written < (int)length) { + int n; + CN1_YIELD_THREAD; + n = SSL_write(ssl, (char*)&data[offset + written], (int)length - written); + CN1_RESUME_THREAD; + if(n <= 0) { + return -2; + } + written += n; + } + return (JAVA_INT)written; +} + +void com_codename1_backend_Tcp_tlsCloseImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG session) { + SSL* ssl = (SSL*)(intptr_t)session; + if(ssl == 0) { + return; + } + /* One shutdown attempt: the descriptor is closed right after this, so waiting + * for the peer's close_notify would only delay it. */ + SSL_shutdown(ssl); + SSL_free(ssl); +} + +#else /* CN1_BACKEND_NO_TLS */ + +JAVA_LONG com_codename1_backend_Tcp_startTlsImpl___long_java_lang_String_java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle, JAVA_OBJECT host, JAVA_OBJECT caFile) { + return 0; +} + +JAVA_OBJECT com_codename1_backend_Tcp_tlsErrorImpl___R_java_lang_String(CODENAME_ONE_THREAD_STATE) { + return newStringFromCString(threadStateData, + "this binary was built without TLS (CN1_BACKEND_HTTPS=0)"); +} + +JAVA_INT com_codename1_backend_Tcp_tlsReadImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG session, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + return -2; +} + +JAVA_INT com_codename1_backend_Tcp_tlsWriteImpl___long_byte_1ARRAY_int_int_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG session, JAVA_OBJECT buffer, JAVA_INT offset, JAVA_INT length) { + return -2; +} + +void com_codename1_backend_Tcp_tlsCloseImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG session) { +} + +#endif diff --git a/vm/backend/native/cn1_backend_web.c b/vm/backend/native/cn1_backend_web.c new file mode 100644 index 00000000000..92db03da6a3 --- /dev/null +++ b/vm/backend/native/cn1_backend_web.c @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/* + * Outbound HTTP and HTTPS for server-side binaries, on libcurl. + * + * Why libcurl rather than the raw-socket client in cn1_backend_net.c: that one is + * plaintext, which is correct for the loopback control protocol it was written for + * and useless for calling anything real. TLS needs a certificate store, hostname + * verification, redirects and chunked decoding, and none of those are things to + * hand-roll into a server that talks to the public internet. This is the same + * choice the native Linux port already made. + * + * Peer and host verification are left at libcurl's defaults (both ON) and there is + * deliberately no knob to turn them off: an "insecure" flag is the kind of thing + * that ships enabled. + */ +#include "cn1_globals.h" +#include +#include +#include +#ifndef _WIN32 +#include /* CN1_RESUME_THREAD expands to usleep */ +#endif +#include + +typedef struct { + char* data; + size_t length; + /* The response headers, verbatim, one per line. Kept as one buffer rather + * than parsed here: the Java side already has to split them, and a header + * parser in C is a second place for the same rules to drift. */ + char* headers; + size_t headerLength; + long status; + char error[CURL_ERROR_SIZE]; +} CN1WebResponse; + +static size_t cn1WebHeader(void* contents, size_t size, size_t count, void* userp) { + CN1WebResponse* r = (CN1WebResponse*)userp; + size_t total = size * count; + char* grown = (char*)realloc(r->headers, r->headerLength + total + 1); + if(grown == NULL) { + return 0; /* tells libcurl to abort the transfer */ + } + r->headers = grown; + memcpy(r->headers + r->headerLength, contents, total); + r->headerLength += total; + r->headers[r->headerLength] = 0; + return total; +} + +/* What a Java byte[] can hold. A response is handed back as one, so a transfer + that outgrows this cannot be delivered however much memory the host has -- + and the length is carried in a size_t here and narrowed to an int there, so + letting it past this point produces a NEGATIVE array length and then a memcpy + of the full size_t into whatever that allocated. Refused while it is still a + failed download, which Web turns into an IOException, rather than after it has + become memory corruption. */ +#define CN1_WEB_MAX_BODY_BYTES ((size_t)0x7fffffff) + +static size_t cn1WebWrite(void* contents, size_t size, size_t count, void* userp) { + CN1WebResponse* r = (CN1WebResponse*)userp; + size_t total = size * count; + if(total > CN1_WEB_MAX_BODY_BYTES - r->length) { + return 0; /* aborts the transfer; libcurl reports CURLE_WRITE_ERROR */ + } + char* grown = (char*)realloc(r->data, r->length + total + 1); + if(grown == NULL) { + return 0; /* tells libcurl to abort the transfer */ + } + r->data = grown; + memcpy(r->data + r->length, contents, total); + r->length += total; + r->data[r->length] = 0; + return total; +} + +/* + * headerLines is one string with '\n' between headers, because passing a + * String[] would mean walking a Java array from C for no benefit. + */ +JAVA_LONG com_codename1_backend_Web_performImpl___java_lang_String_java_lang_String_java_lang_String_byte_1ARRAY_R_long( + CODENAME_ONE_THREAD_STATE, JAVA_OBJECT method, JAVA_OBJECT url, JAVA_OBJECT headerLines, JAVA_OBJECT body) { + CURL* curl; + CURLcode rc; + struct curl_slist* headers = NULL; + CN1WebResponse* r; + char* methodCopy = NULL; + char* urlCopy = NULL; + char* bodyCopy = NULL; + JAVA_INT bodyLength = 0; + + if(url == JAVA_NULL) { + return 0; + } + /* stringToUTF8 returns this thread's scratch buffer, which the NEXT conversion + overwrites -- so every string is copied before the next one is converted. */ + { + const char* tmp = stringToUTF8(threadStateData, url); + if(tmp == NULL) { + return 0; + } + urlCopy = strdup(tmp); + } + if(method != JAVA_NULL) { + const char* tmp = stringToUTF8(threadStateData, method); + methodCopy = tmp == NULL ? NULL : strdup(tmp); + } + if(headerLines != JAVA_NULL) { + const char* tmp = stringToUTF8(threadStateData, headerLines); + if(tmp != NULL && tmp[0] != 0) { + char* copy = strdup(tmp); + char* line = copy; + while(line != NULL && *line != 0) { + char* nl = strchr(line, '\n'); + if(nl != NULL) { + *nl = 0; + } + if(*line != 0) { + headers = curl_slist_append(headers, line); + } + line = nl == NULL ? NULL : nl + 1; + } + free(copy); + } + } + if(body != JAVA_NULL) { + JAVA_ARRAY arr = (JAVA_ARRAY)body; + bodyLength = arr->length; + bodyCopy = (char*)malloc(bodyLength == 0 ? 1 : (size_t)bodyLength); + if(bodyCopy == NULL && bodyLength > 0) { + /* The request must NOT go out without it. The POSTFIELDS block below + is skipped when bodyCopy is null, so a failed allocation sent the + same request with an EMPTY body and reported success: an S3 + putObject would replace the object with nothing, and the caller + would be told it worked. A request whose body could not be made is + a failed request. */ + free(urlCopy); + free(methodCopy); + curl_slist_free_all(headers); + return 0; + } + if(bodyCopy != NULL && bodyLength > 0) { + memcpy(bodyCopy, (JAVA_ARRAY_BYTE*)arr->data, (size_t)bodyLength); + } + } + + r = (CN1WebResponse*)calloc(1, sizeof(CN1WebResponse)); + if(r == NULL) { + free(urlCopy); free(methodCopy); free(bodyCopy); + curl_slist_free_all(headers); + return 0; + } + + curl = curl_easy_init(); + if(curl == NULL) { + free(r); free(urlCopy); free(methodCopy); free(bodyCopy); + curl_slist_free_all(headers); + return 0; + } + curl_easy_setopt(curl, CURLOPT_URL, urlCopy); + /* The path goes out exactly as the caller wrote it. + libcurl otherwise resolves "." and ".." before sending, while the SigV4 + signature was computed over the UNNORMALISED path -- so an S3 key with a dot + segment in it is signed for one path and requested at another, and comes back + SignatureDoesNotMatch. The JavaSE path does not normalise, so such a key works + under cn1:backend and fails only once packaged. */ +/* Guarded on the VERSION, not on #ifdef. CURLOPT_PATH_AS_IS is an enum member + -- curl.h declares it as CURLOPT(CURLOPT_PATH_AS_IS, CURLOPTTYPE_LONG, 234), + not as a macro -- so the preprocessor has never heard of it and the #ifdef + this replaces was always false. The option was therefore never set, on any + libcurl, and the guard read as if it were. + What that costs: libcurl normalises dot segments, so a request for an S3 key + holding "a/../b" goes out as "/b" while Aws signed "/a/../b", and the service + answers SignatureDoesNotMatch. The Java SE arm sends the path as written, so + the key works under cn1:backend and fails once packaged -- which is exactly + the divergence the comment above claims to prevent. + 7.42.0 is where the option appeared. */ +#if LIBCURL_VERSION_NUM >= 0x072A00 + curl_easy_setopt(curl, CURLOPT_PATH_AS_IS, 1L); +#endif + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cn1WebWrite); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, r); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, cn1WebHeader); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, r); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, r->error); + /* Following a redirect RESENDS the caller's headers to wherever it points. + libcurl drops Authorization when the host changes, but it knows nothing + about X-Api-Key, X-Amz-Security-Token or any other bearer a caller + invented, and those go to the new host in full. A 3xx from a service that + has been taken over, or simply one that redirects off-domain, is then + enough to hand an attacker the credential -- the caller never sees where + its header went. + So redirects are followed freely only when the caller supplied NO headers, + where there is nothing to leak. With headers, following is restricted to + the same host where libcurl can express that, and otherwise not done at + all: the 3xx and its Location are returned as the response, which the + caller can act on deliberately. */ + if(headers == NULL) { + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + } else { + /* This #ifdef may never fire, for the same reason the PATH_AS_IS one + above did not: if libcurl declares CURLFOLLOW_SAMEHOST through its + CURLOPT-style enum rather than as a macro, the preprocessor cannot see + it. Left as it is deliberately, because the two fail in OPPOSITE + directions. There, a guard that never fires left the option off and + the request wrong; here it selects the #else, which does not follow + the redirect at all -- more restrictive than intended, and still the + safe answer, since the whole point is not to carry the caller's + headers to another host. A version guard is not written for it because + the release that introduced the constant cannot be checked from here; + an unverified version number would be a worse guess than this. */ +#ifdef CURLFOLLOW_SAMEHOST + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, (long)CURLFOLLOW_SAMEHOST); +#else + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0L); +#endif + } + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L); + /* A CONNECT deadline and a STALL deadline, not a deadline on the whole + transfer. CURLOPT_TIMEOUT caps the entire operation, so a large upload or + download that is progressing perfectly well is aborted at 30 seconds for + no reason other than its size -- and the Java SE arm does not do that: it + sets a READ timeout, which fires only when a single read stalls. The two + have to agree, or an S3 object big enough to take half a minute transfers + under cn1:backend and fails once packaged. + LOW_SPEED_LIMIT/LOW_SPEED_TIME is libcurl's spelling of the same idea: + give up when the transfer makes essentially no progress for 30s. */ + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 30L); + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "codenameone-backend"); + /* CN1_WEB_VERBOSE=1 makes libcurl narrate the exchange on stderr. Off by + * default and read per request rather than cached, so it can be turned on for + * a running process through its environment without a rebuild. Request headers + * carry credentials, so this is a debugging switch, not a logging one. */ + if(getenv("CN1_WEB_VERBOSE") != NULL) { + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L); + } + if(headers != NULL) { + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + } + if(methodCopy != NULL) { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, methodCopy); + /* CUSTOMREQUEST only changes the METHOD WORD. For HEAD that is not + * enough: libcurl still expects a response body, so it reads the + * Content-Length the server reports for the entity it is NOT sending and + * waits for bytes that never arrive -- a hang until CURLOPT_TIMEOUT, with + * "0 out of N bytes received". NOBODY is what tells it the response ends + * at the headers. Found by S3.headObject, which is the first HEAD this + * client ever sent. */ + if(strcmp(methodCopy, "HEAD") == 0) { + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + } + } + if(bodyCopy != NULL) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, bodyCopy); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, (long)bodyLength); + } + + /* The transfer blocks; yield so the concurrent collector is not stalled by it. */ + CN1_YIELD_THREAD; + rc = curl_easy_perform(curl); + CN1_RESUME_THREAD; + + if(rc == CURLE_OK) { + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &r->status); + } else { + r->status = -1; + if(r->error[0] == 0) { + const char* msg = curl_easy_strerror(rc); + strncpy(r->error, msg == NULL ? "transfer failed" : msg, CURL_ERROR_SIZE - 1); + } + } + curl_easy_cleanup(curl); + curl_slist_free_all(headers); + free(urlCopy); free(methodCopy); free(bodyCopy); + return (JAVA_LONG)(intptr_t)r; +} + +JAVA_INT com_codename1_backend_Web_statusImpl___long_R_int(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1WebResponse* r = (CN1WebResponse*)(intptr_t)handle; + return r == NULL ? -1 : (JAVA_INT)r->status; +} + +JAVA_OBJECT com_codename1_backend_Web_errorImpl___long_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1WebResponse* r = (CN1WebResponse*)(intptr_t)handle; + if(r == NULL || r->error[0] == 0) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, r->error); +} + +JAVA_OBJECT com_codename1_backend_Web_bodyImpl___long_R_byte_1ARRAY(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1WebResponse* r = (CN1WebResponse*)(intptr_t)handle; + JAVA_OBJECT arr; + if(r == NULL) { + return JAVA_NULL; + } + if(r->length > CN1_WEB_MAX_BODY_BYTES) { + /* Unreachable while cn1WebWrite holds the line above, and checked anyway: + the cast below is what turns a length this size into a negative one, + and the memcpy after it does not consult the array's length. */ + return JAVA_NULL; + } + arr = allocArray(threadStateData, (int)r->length, &class_array1__JAVA_BYTE, sizeof(JAVA_ARRAY_BYTE), 1); + if(r->length > 0 && r->data != NULL) { + memcpy((JAVA_ARRAY_BYTE*)((JAVA_ARRAY)arr)->data, r->data, r->length); + } + return arr; +} + +JAVA_OBJECT com_codename1_backend_Web_headersImpl___long_R_java_lang_String(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1WebResponse* r = (CN1WebResponse*)(intptr_t)handle; + if(r == NULL || r->headers == NULL) { + return JAVA_NULL; + } + return newStringFromCString(threadStateData, r->headers); +} + +JAVA_VOID com_codename1_backend_Web_freeImpl___long(CODENAME_ONE_THREAD_STATE, JAVA_LONG handle) { + CN1WebResponse* r = (CN1WebResponse*)(intptr_t)handle; + if(r != NULL) { + free(r->data); + free(r->headers); + free(r); + } +} diff --git a/vm/backend/package.sh b/vm/backend/package.sh new file mode 100755 index 00000000000..2785de606f0 --- /dev/null +++ b/vm/backend/package.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Builds one backend program for every Linux deployment target. +# +# package.sh [target...] +# +# Targets are -: musl-x86_64, musl-arm64, glibc-x86_64, glibc-arm64. +# With none named it builds all four. Output lands in target/dist/. +# +# The translation runs ONCE. ParparVM emits portable C, so what differs between +# targets is only the compile -- which is why this is four clang invocations over +# one source tree rather than four builds. +# +# Why both libcs: +# musl a fully static binary: no libc, no OpenSSL, nothing. It runs in a +# scratch or distroless image, so the container is the binary and there +# is no base image to patch. This is the microservice shape. +# glibc linked against the distribution's libc and OpenSSL, for an +# organisation whose base image already carries them and patches them on +# its own schedule. +# +# Cross-architecture builds go through qemu (podman/docker emulate the other +# arch), which works and is slow. On a build machine, prefer a native runner per +# architecture and name one target. +# +# Environment knobs: +# CN1_BACKEND_DEMO demo source dir (default demo/petserver) +# CN1_BACKEND_ENGINE podman or docker (default: whichever is on PATH) +# CN1_BACKEND_SQLITE=0 leave the SQLite engine out +# CN1_BACKEND_HTTPS=0 leave TLS and outbound HTTP out +set -e +cd "$(dirname "$0")" +MAIN="${1:?usage: package.sh [target...]}"; shift +PKG="${1:?usage: package.sh [target...]}"; shift +TARGETS="$*" +if [ -z "$TARGETS" ]; then + TARGETS="musl-x86_64 musl-arm64 glibc-x86_64 glibc-arm64" +fi + +ENGINE="${CN1_BACKEND_ENGINE:-}" +if [ -z "$ENGINE" ]; then + for candidate in podman docker; do + if command -v "$candidate" >/dev/null 2>&1; then ENGINE="$candidate"; break; fi + done +fi +[ -n "$ENGINE" ] || { echo "no container engine found; install podman or docker"; exit 1; } + +SRC="$(pwd)/target/csrc-$MAIN" +DIST="$(pwd)/target/dist" +mkdir -p "$DIST" + +# One translation for every target. +CN1_BACKEND_SRC_OUT="$SRC" ./build.sh "$MAIN" "$PKG" unused +# build.sh derives the -D flags that go with the switches it was given and leaves +# them beside the sources; the container link runs in its own process and would +# otherwise link a source tree it has not been told about. +DERIVED_CFLAGS="" +if [ -f "$SRC/cn1-cflags.txt" ]; then + DERIVED_CFLAGS="$(cat "$SRC/cn1-cflags.txt")" + rm -f "$SRC/cn1-cflags.txt" +fi + +lower() { echo "$1" | tr 'A-Z' 'a-z'; } + +for target in $TARGETS; do + libc="${target%%-*}" + arch="${target#*-}" + case "$libc" in + musl|glibc) ;; + *) echo "unknown libc in target '$target' (expected musl or glibc)"; exit 1 ;; + esac + case "$arch" in + x86_64) platform="linux/amd64" ;; + arm64) platform="linux/arm64" ;; + *) echo "unknown architecture in target '$target' (expected x86_64 or arm64)"; exit 1 ;; + esac + + image="cn1-backend-$libc-$arch" + echo "==> building the $libc/$arch builder image" + "$ENGINE" build --platform "$platform" -t "$image" \ + -f "docker/Containerfile.$libc" docker + + out_name="$(lower "$MAIN")-linux-$libc-$arch" + # Removed first: a failed link would otherwise leave the PREVIOUS binary in + # place, and a stale artifact that looks fresh is worse than no artifact. + rm -f "$DIST/$out_name" + echo "==> linking $out_name" + "$ENGINE" run --rm --platform "$platform" \ + -v "$SRC:/src:ro,Z" -v "$DIST:/out:Z" \ + -e "CN1_OUT_NAME=$out_name" \ + -e "CN1_EXTRA_CFLAGS=$DERIVED_CFLAGS $CN1_BACKEND_CFLAGS" \ + -e "CN1_LINK_DEBUG=${CN1_LINK_DEBUG:-}" \ + "$image" +done + +echo +echo "built:" +ls -l "$DIST" | tail -n +2 | sed 's/^/ /' diff --git a/vm/backend/parity-check.sh b/vm/backend/parity-check.sh new file mode 100755 index 00000000000..d91653c91a1 --- /dev/null +++ b/vm/backend/parity-check.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Proves the local Java SE runtime and the native binary answer the same. +# +# The shared runtime (src/) is one copy of the protocol logic, so it cannot drift +# on its own -- but the per-target impl/ classes underneath it can, and a dev loop +# that behaves differently from production is worse than no dev loop. This runs the +# SAME request script against both and diffs the answers. +# +# Both are exercised over a real socket, not in-process, so what is compared is +# what a client sees: status lines, headers that matter, and bodies. +set -e +cd "$(dirname "$0")" +PORT_JVM="${CN1_PARITY_PORT_JVM:-8471}" +PORT_NATIVE="${CN1_PARITY_PORT_NATIVE:-8472}" +OUT="target/parity" +rm -rf "$OUT"; mkdir -p "$OUT" + +# Every response goes through this so the parts that are ALLOWED to differ do not +# register as drift: a JWT carries an issued-at and a random-per-process signing +# secret, and Date is a wall clock. +normalize() { + sed -E -e 's/eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+//g' \ + -e 's/^Date:.*/Date: /' \ + -e 's/"uptimeSeconds":[0-9]+/"uptimeSeconds":/' \ + -e 's/^Server:.*/Server: /' \ + -e 's/\r$//' +} + +# One request per line, run in order against whichever server is up. Bodies that +# depend on a token use $TOK, which the script fills in from /login. +probe() { + local base="$1" out="$2" + local tok + tok="$(curl -sS --max-time 10 -X POST "$base/login" -H 'Content-Type: application/json' \ + -d '{"username":"shai","password":"hunter2"}' | tr -d '"')" + { + echo "### greet"; curl -sS --max-time 10 "$base/greet/Shai"; echo + echo "### greet loud"; curl -sS --max-time 10 "$base/greet/Shai?loud=yes"; echo + echo "### whoami"; curl -sS --max-time 10 "$base/whoami" -H 'X-User: shai' \ + -H 'Cookie: session=abc123'; echo + echo "### login bad"; curl -sS --max-time 10 -X POST "$base/login" \ + -H 'Content-Type: application/json' \ + -d '{"username":"shai","password":"wrong"}'; echo + echo "### login shape"; echo "$tok" | cut -c1-3; echo + echo "### addPet"; curl -sS --max-time 10 -X POST "$base/pet" \ + -H 'Content-Type: application/json' \ + -d '{"name":"Rex","species":"dog","weight":12.5,"good":true}'; echo + echo "### getPet"; curl -sS --max-time 10 "$base/pet/1"; echo + echo "### getPet 404"; curl -sS --max-time 10 -o /dev/null -w '%{http_code}\n' "$base/pet/999" + echo "### bulk"; curl -sS --max-time 10 -X POST "$base/pets/bulk" \ + -H "Authorization: Bearer $tok" \ + -H 'Content-Type: application/json' \ + -d '[{"name":"Mia","species":"cat"},{"name":"Bo","species":"dog"}]'; echo + echo "### bulk no auth"; curl -sS --max-time 10 -X POST "$base/pets/bulk" \ + -H 'Content-Type: application/json' -d '[]'; echo + echo "### bulk bad tok"; curl -sS --max-time 10 -X POST "$base/pets/bulk" \ + -H "Authorization: Bearer ${tok%?}X" \ + -H 'Content-Type: application/json' -d '[]'; echo + echo "### listPets"; curl -sS --max-time 10 "$base/pets?species=dog"; echo + echo "### echo nested"; curl -sS --max-time 10 -X POST "$base/echo" \ + -H 'Content-Type: application/json' \ + -d '{"name":"Rex","species":"dog","weight":1.5,"good":true,"tags":[{"label":"friendly","weight":3},{"label":"loud","weight":1}]}'; echo + echo "### echo bad nested"; curl -sS --max-time 10 -X POST "$base/echo" \ + -H 'Content-Type: application/json' \ + -d '{"name":"Rex","tags":["not-an-object",7]}'; echo + echo "### echo array body"; curl -sS --max-time 10 -X POST "$base/echo" \ + -H 'Content-Type: application/json' -d '[1,2,3]'; echo + echo "### bulk not array"; curl -sS --max-time 10 -X POST "$base/pets/bulk" \ + -H "Authorization: Bearer $tok" \ + -H 'Content-Type: application/json' -d '{"name":"x"}'; echo + echo "### photo set"; curl -sS --max-time 10 -X POST "$base/pet/1/photo" \ + --data-binary 'aGVsbG8='; echo + echo "### photo get"; curl -sS --max-time 10 "$base/pet/1/photo"; echo + echo "### delete"; curl -sS --max-time 10 -X DELETE "$base/pet/2" \ + -H "Authorization: Bearer $tok"; echo + echo "### list after"; curl -sS --max-time 10 "$base/pets"; echo + echo "### healthz"; curl -sS --max-time 10 "$base/healthz"; echo + echo "### unknown"; curl -sS --max-time 10 -o /dev/null -w '%{http_code}\n' "$base/nope" + echo "### bad method"; curl -sS --max-time 10 -o /dev/null -w '%{http_code}\n' \ + -X PUT "$base/pet/1" + echo "### bad json"; curl -sS --max-time 10 -X POST "$base/pet" \ + -H 'Content-Type: application/json' -d '{not json'; echo + echo "### keepalive"; curl -sS --max-time 10 "$base/greet/a" "$base/greet/b"; echo + echo "### headers"; curl -sS --max-time 10 -D - -o /dev/null "$base/pet/1" + } 2>&1 | normalize > "$out" +} + +wait_for() { + local base="$1" tries=0 + while [ "$tries" -lt 100 ]; do + if curl -sS --max-time 2 -o /dev/null "$base/pets" 2>/dev/null; then return 0; fi + tries=$((tries + 1)) + sleep 0.2 + done + echo "server never came up at $base"; return 1 +} + +DB_JVM="$(mktemp "${TMPDIR:-/tmp}/cn1parity-jvm.XXXXXX")" +DB_NATIVE="$(mktemp "${TMPDIR:-/tmp}/cn1parity-native.XXXXXX")" +rm -f "$DB_JVM" "$DB_NATIVE" + +CN1_BACKEND_DEMO=demo/petserver CN1_PORT="$PORT_JVM" CN1_DB_PATH="$DB_JVM" \ + ./run-javase.sh com.demo.PetServer > "$OUT/jvm.log" 2>&1 & +JVM_PID=$! +trap 'kill $JVM_PID 2>/dev/null; kill $NATIVE_PID 2>/dev/null' EXIT + +# Rebuilt every run by default. A binary left over from an earlier tree would +# make this compare today's Java SE runtime against last week's native one and +# call the agreement proof of anything. CN1_PARITY_REUSE_BINARY=1 keeps it while +# iterating on the Java SE side. +if [ "${CN1_PARITY_REUSE_BINARY:-0}" != "1" ] || [ ! -x target/petserver-native ]; then + CN1_BACKEND_DEMO=demo/petserver ./build.sh PetServer com.demo target/petserver-native +fi +CN1_PORT="$PORT_NATIVE" CN1_DB_PATH="$DB_NATIVE" \ + ./target/petserver-native > "$OUT/native.log" 2>&1 & +NATIVE_PID=$! + +wait_for "http://127.0.0.1:$PORT_JVM" +wait_for "http://127.0.0.1:$PORT_NATIVE" +probe "http://127.0.0.1:$PORT_JVM" "$OUT/jvm.txt" +probe "http://127.0.0.1:$PORT_NATIVE" "$OUT/native.txt" + +if diff -u "$OUT/native.txt" "$OUT/jvm.txt" > "$OUT/diff.txt"; then + echo "PARITY OK -- $(grep -c '^###' "$OUT/jvm.txt") probes identical on both runtimes" +else + echo "PARITY FAILED -- the two runtimes answered differently:" + cat "$OUT/diff.txt" + exit 1 +fi diff --git a/vm/backend/run-javase.sh b/vm/backend/run-javase.sh new file mode 100755 index 00000000000..84ee71bc131 --- /dev/null +++ b/vm/backend/run-javase.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Runs a backend demo on a plain JVM, for the fast local edit-run loop. +# +# run-javase.sh [program args...] +# +# The SAME shared runtime (src/) that the native build translates is compiled here; +# only impl/ differs -- impl/javase instead of impl/parparvm. That is the whole +# point of the split: protocol behaviour cannot drift between the loop you develop +# in and the binary you ship, because there is one copy of it. +# +# What the local runtime deliberately does NOT do: terminate TLS, and therefore +# serve HTTP/2 (Tls and Http2 say so and refuse). Run build.sh for those. +# +# Environment knobs: +# CN1_BACKEND_DEMO demo source dir (default demo/petserver) +# CN1_BACKEND_JDBC_JARS extra classpath entries for JDBC drivers +# CN1_BACKEND_JAVA the java/javac home to use (default: JAVA17_HOME, then PATH) +set -e +cd "$(dirname "$0")" +MAIN="${1:?usage: run-javase.sh [args...]}"; shift + +JAVA_HOME_DIR="${CN1_BACKEND_JAVA:-${JAVA17_HOME:-}}" +if [ -n "$JAVA_HOME_DIR" ] && [ -x "$JAVA_HOME_DIR/bin/javac" ]; then + JAVAC="$JAVA_HOME_DIR/bin/javac"; JAVA="$JAVA_HOME_DIR/bin/java" +else + JAVAC="$(command -v javac)"; JAVA="$(command -v java)" +fi +[ -x "$JAVAC" ] || { echo "no javac found; set CN1_BACKEND_JAVA or JAVA17_HOME"; exit 1; } + +DEMO="${CN1_BACKEND_DEMO:-demo/petserver}" +[ -d "$DEMO" ] || { echo "demo directory not found: $DEMO"; exit 1; } +COMMON="" +if [ -d demo/common ]; then COMMON="demo/common"; fi +# gen/ holds COMPILED classes from generate-contract.sh (the server half of the +# shared @RestClient contract), so it goes on the classpath rather than the source +# list -- exactly as build.sh treats it. +if [ -d contract ]; then ./generate-contract.sh --if-needed; fi +GEN="" +if [ -d gen ]; then GEN="gen"; fi + +# JDBC drivers are optional: without one, Db.open fails with a message that says +# so, and everything that does not touch a database still runs. sqlite-jdbc needs +# slf4j-api on the classpath as well -- without it the driver's service entry +# throws while being instantiated and DriverManager reports "no suitable driver", +# which names neither the real cause nor the missing jar. +newest_jar() { + ls -1 "$HOME/.m2/repository/$1/$2/"*/"$2"-*.jar 2>/dev/null \ + | grep -v -- '-sources\.jar$' | grep -v -- '-javadoc\.jar$' \ + | sort -V | tail -1 +} +DRIVERS="$CN1_BACKEND_JDBC_JARS" +if [ -z "$DRIVERS" ]; then + for jar in $(newest_jar org/xerial sqlite-jdbc) $(newest_jar org/slf4j slf4j-api); do + if [ -z "$DRIVERS" ]; then DRIVERS="$jar"; else DRIVERS="$DRIVERS:$jar"; fi + done +fi + +# A private output directory per run, removed when the JVM exits. +# +# It used to be one shared target/javase-classes that every run deleted and +# rebuilt, which is fine until two runs overlap -- the test suite forks several, +# and the loser's javac fails with "directory not found" on a directory the +# winner removed out from under it. A build this cheap is not worth sharing. +mkdir -p target +OUT="$(mktemp -d "$(pwd)/target/javase.XXXXXX")" +trap 'rm -rf "$OUT"' EXIT +BUILD_CP="$OUT" +if [ -n "$GEN" ]; then BUILD_CP="$BUILD_CP:$GEN"; fi +if [ -n "$DRIVERS" ]; then BUILD_CP="$BUILD_CP:$DRIVERS"; fi +"$JAVAC" -nowarn -encoding UTF-8 -cp "$BUILD_CP" -d "$OUT" \ + $(find src impl/javase $COMMON "$DEMO" -name '*.java') +if [ -n "$GEN" ]; then cp -r "$GEN/." "$OUT/"; fi + +CP="$OUT" +if [ -n "$DRIVERS" ]; then CP="$CP:$DRIVERS"; fi +# Not exec: the trap above has to run so the class directory does not accumulate. +# The JVM is in this shell's process group, so Ctrl-C still reaches it. +"$JAVA" -cp "$CP" "$MAIN" "$@" diff --git a/vm/backend/src/com/codename1/backend/Base64.java b/vm/backend/src/com/codename1/backend/Base64.java new file mode 100644 index 00000000000..c15cd22213a --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Base64.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * Standard base64 (RFC 4648 section 4), with padding. + * + * Not {@link Base64Url}: that one is the URL-safe alphabet with the padding + * stripped, because that is what JWT specifies. These two alphabets are not + * interchangeable, and the protocols that need this one -- SCRAM-SHA-256 in the + * PostgreSQL handshake, and AWS request signing -- reject the other. + * + * Decoding is strict about length and alphabet. A lenient decoder is how a + * signature comparison ends up accepting more than one encoding of the same + * bytes; see the note in {@link Base64Url}. + */ +public final class Base64 { + private static final char[] ALPHABET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray(); + private static final int[] REVERSE = new int[128]; + + static { + for(int iter = 0 ; iter < REVERSE.length ; iter++) { + REVERSE[iter] = -1; + } + for(int iter = 0 ; iter < ALPHABET.length ; iter++) { + REVERSE[ALPHABET[iter]] = iter; + } + } + + private Base64() { + } + + public static String encode(byte[] data) { + if(data == null) { + return null; + } + StringBuilder out = new StringBuilder(((data.length + 2) / 3) * 4); + int iter = 0; + while(iter + 2 < data.length) { + int block = ((data[iter] & 0xff) << 16) | ((data[iter + 1] & 0xff) << 8) + | (data[iter + 2] & 0xff); + out.append(ALPHABET[(block >> 18) & 0x3f]).append(ALPHABET[(block >> 12) & 0x3f]) + .append(ALPHABET[(block >> 6) & 0x3f]).append(ALPHABET[block & 0x3f]); + iter += 3; + } + int remaining = data.length - iter; + if(remaining == 1) { + int block = (data[iter] & 0xff) << 16; + out.append(ALPHABET[(block >> 18) & 0x3f]).append(ALPHABET[(block >> 12) & 0x3f]) + .append('=').append('='); + } else if(remaining == 2) { + int block = ((data[iter] & 0xff) << 16) | ((data[iter + 1] & 0xff) << 8); + out.append(ALPHABET[(block >> 18) & 0x3f]).append(ALPHABET[(block >> 12) & 0x3f]) + .append(ALPHABET[(block >> 6) & 0x3f]).append('='); + } + return out.toString(); + } + + /** The decoded bytes, or null when the input is not valid base64. */ + public static byte[] decode(String value) { + if(value == null || (value.length() % 4) != 0) { + return null; + } + int padding = 0; + int length = value.length(); + while(padding < 2 && length - padding > 0 && value.charAt(length - padding - 1) == '=') { + padding++; + } + int bytes = (length / 4) * 3 - padding; + byte[] out = new byte[bytes]; + int at = 0; + for(int iter = 0 ; iter < length ; iter += 4) { + int block = 0; + int pads = 0; + for(int part = 0 ; part < 4 ; part++) { + char c = value.charAt(iter + part); + if(c == '=') { + // Padding is only legal in the final group, and only where the + // length says it should be. + if(iter + 4 != length || part < 2) { + return null; + } + pads++; + block <<= 6; + continue; + } + // CONTIGUOUS, and at the end. "AA=A" satisfied both tests above -- + // final group, '=' at index 2 -- and then took the 'A' after it as + // ordinary data, returning three bytes for a string no encoder can + // produce. Once padding starts the group is over. + if(pads > 0) { + return null; + } + if(c >= REVERSE.length || REVERSE[c] < 0) { + return null; + } + block = (block << 6) | REVERSE[c]; + } + // The bits the padding stands for have to be zero, or one byte sequence + // has several spellings -- "AB==" and "AA==" would both decode to a + // single 0 byte -- which a strict decoder must not accept. + if(pads == 2 && ((block >> 12) & 0x0f) != 0) { + return null; + } + if(pads == 1 && ((block >> 6) & 0x03) != 0) { + return null; + } + for(int part = 16 ; part >= 0 && at < bytes ; part -= 8) { + out[at++] = (byte)((block >> part) & 0xff); + } + } + return out; + } +} diff --git a/vm/backend/src/com/codename1/backend/Base64Url.java b/vm/backend/src/com/codename1/backend/Base64Url.java new file mode 100644 index 00000000000..f05a1585396 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Base64Url.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * Base64url without padding, as JSON Web Tokens use it. Separate from any general + * base64 because the alphabet differs ('-' and '_' for '+' and '/') and a token + * encoded with the wrong one is rejected by every other implementation. + */ +public final class Base64Url { + private static final char[] ALPHABET = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".toCharArray(); + + private Base64Url() { + } + + public static String encode(byte[] data) { + if(data == null) { + return null; + } + StringBuilder out = new StringBuilder((data.length + 2) / 3 * 4); + int iter = 0; + while(iter + 2 < data.length) { + int n = ((data[iter] & 0xff) << 16) | ((data[iter + 1] & 0xff) << 8) | (data[iter + 2] & 0xff); + out.append(ALPHABET[(n >>> 18) & 63]).append(ALPHABET[(n >>> 12) & 63]) + .append(ALPHABET[(n >>> 6) & 63]).append(ALPHABET[n & 63]); + iter += 3; + } + int remaining = data.length - iter; + if(remaining == 1) { + int n = (data[iter] & 0xff) << 16; + out.append(ALPHABET[(n >>> 18) & 63]).append(ALPHABET[(n >>> 12) & 63]); + } else if(remaining == 2) { + int n = ((data[iter] & 0xff) << 16) | ((data[iter + 1] & 0xff) << 8); + out.append(ALPHABET[(n >>> 18) & 63]).append(ALPHABET[(n >>> 12) & 63]) + .append(ALPHABET[(n >>> 6) & 63]); + } + return out.toString(); + } + + /** Null for anything that is not valid base64url, rather than a partial result. */ + public static byte[] decode(String value) { + if(value == null) { + return null; + } + int length = value.length(); + int fullGroups = length / 4; + int remaining = length % 4; + if(remaining == 1) { + return null; // no valid encoding leaves a single character over + } + int size = fullGroups * 3 + (remaining == 0 ? 0 : remaining - 1); + byte[] out = new byte[size]; + int outPos = 0; + int buffer = 0; + int bits = 0; + for(int iter = 0 ; iter < length ; iter++) { + int v = valueOf(value.charAt(iter)); + if(v < 0) { + return null; + } + buffer = (buffer << 6) | v; + bits += 6; + if(bits >= 8) { + bits -= 8; + if(outPos >= size) { + return null; + } + out[outPos++] = (byte)((buffer >>> bits) & 0xff); + } + } + // The leftover bits of the final character must be zero. Accepting a + // non-canonical encoding means several distinct strings decode to the same + // bytes -- for a JWT that is token malleability: an attacker can hand back + // a different-looking token that still verifies, which breaks anything + // keyed on the token string, a revocation list most of all. + if(bits > 0 && (buffer & ((1 << bits) - 1)) != 0) { + return null; + } + return outPos == size ? out : null; + } + + private static int valueOf(char c) { + if(c >= 'A' && c <= 'Z') { + return c - 'A'; + } + if(c >= 'a' && c <= 'z') { + return c - 'a' + 26; + } + if(c >= '0' && c <= '9') { + return c - '0' + 52; + } + if(c == '-') { + return 62; + } + if(c == '_') { + return 63; + } + return -1; + } +} diff --git a/vm/backend/src/com/codename1/backend/ByteSink.java b/vm/backend/src/com/codename1/backend/ByteSink.java new file mode 100644 index 00000000000..59d28a9794e --- /dev/null +++ b/vm/backend/src/com/codename1/backend/ByteSink.java @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * A growable byte buffer that callers reuse. + * + * This exists because building output as a String and then encoding it is the + * single most expensive thing a server can do per request. Measured on this + * server: the response head alone, built with a StringBuilder, turned into a + * String and then into bytes, was a third of all allocation; the JSON body was + * most of the rest. Written as bytes into a buffer that lives as long as the + * connection, both cost nothing. + * + * Deliberately not java.io.ByteArrayOutputStream: that one cannot be reset + * without discarding its buffer on some implementations, has synchronized + * methods, and hands out a COPY of its contents -- three allocations where this + * has none. + * + * Not thread safe. One per connection, used by the worker that owns it. + */ +public final class ByteSink { + private byte[] data; + private int length; + + public ByteSink(int initialCapacity) { + data = new byte[initialCapacity < 16 ? 16 : initialCapacity]; + } + + /** The backing array. Valid up to {@link #length}; not a copy. */ + public byte[] bytes() { + return data; + } + + public int length() { + return length; + } + + public void reset() { + length = 0; + } + + public void ensure(int extra) { + if(length + extra <= data.length) { + return; + } + int size = data.length * 2; + while(size < length + extra) { + size *= 2; + } + byte[] grown = new byte[size]; + System.arraycopy(data, 0, grown, 0, length); + data = grown; + } + + public void put(int b) { + ensure(1); + data[length++] = (byte)b; + } + + public void put(byte[] source, int offset, int count) { + ensure(count); + System.arraycopy(source, offset, data, length, count); + length += count; + } + + public void put(ByteSink other) { + put(other.data, 0, other.length); + } + + /** + * ASCII only, one byte per character. For header names, JSON punctuation and + * other text this code owns; anything from outside goes through + * {@link #putUtf8}. + */ + public void putAscii(String ascii) { + int n = ascii.length(); + ensure(n); + for(int iter = 0 ; iter < n ; iter++) { + data[length++] = (byte)ascii.charAt(iter); + } + } + + /** + * UTF-8, encoded in place. + * + * String.getBytes("UTF-8") would allocate the array this exists to avoid, and + * on the translated target it goes through the platform's encoder for every + * call. Surrogate pairs are combined; an unpaired surrogate becomes U+FFFD, + * because emitting a lone surrogate produces bytes no decoder will accept. + */ + public void putUtf8(String value) { + int n = value.length(); + ensure(n); // exact for ASCII, grown below otherwise + for(int iter = 0 ; iter < n ; iter++) { + int c = value.charAt(iter); + if(c < 0x80) { + ensure(1); + data[length++] = (byte)c; + } else if(c < 0x800) { + ensure(2); + data[length++] = (byte)(0xc0 | (c >> 6)); + data[length++] = (byte)(0x80 | (c & 0x3f)); + } else if(c >= 0xd800 && c <= 0xdbff && iter + 1 < n + && value.charAt(iter + 1) >= 0xdc00 && value.charAt(iter + 1) <= 0xdfff) { + int code = 0x10000 + ((c - 0xd800) << 10) + (value.charAt(iter + 1) - 0xdc00); + iter++; + ensure(4); + data[length++] = (byte)(0xf0 | (code >> 18)); + data[length++] = (byte)(0x80 | ((code >> 12) & 0x3f)); + data[length++] = (byte)(0x80 | ((code >> 6) & 0x3f)); + data[length++] = (byte)(0x80 | (code & 0x3f)); + } else if(c >= 0xd800 && c <= 0xdfff) { + ensure(3); // unpaired surrogate -> U+FFFD + data[length++] = (byte)0xef; + data[length++] = (byte)0xbf; + data[length++] = (byte)0xbd; + } else { + ensure(3); + data[length++] = (byte)(0xe0 | (c >> 12)); + data[length++] = (byte)(0x80 | ((c >> 6) & 0x3f)); + data[length++] = (byte)(0x80 | (c & 0x3f)); + } + } + } + + /** + * One code point as UTF-8. + * + * Separate from {@link #putUtf8} because a caller walking a String character + * by character has to combine a surrogate PAIR itself -- handing the halves + * over one at a time turns an emoji into two replacement characters, which is + * what the JSON writer did until its output was compared against the String + * form byte for byte. + */ + public void putCodePoint(int code) { + if(code < 0x80) { + ensure(1); + data[length++] = (byte)code; + } else if(code < 0x800) { + ensure(2); + data[length++] = (byte)(0xc0 | (code >> 6)); + data[length++] = (byte)(0x80 | (code & 0x3f)); + } else if(code < 0x10000) { + ensure(3); + data[length++] = (byte)(0xe0 | (code >> 12)); + data[length++] = (byte)(0x80 | ((code >> 6) & 0x3f)); + data[length++] = (byte)(0x80 | (code & 0x3f)); + } else { + ensure(4); + data[length++] = (byte)(0xf0 | (code >> 18)); + data[length++] = (byte)(0x80 | ((code >> 12) & 0x3f)); + data[length++] = (byte)(0x80 | ((code >> 6) & 0x3f)); + data[length++] = (byte)(0x80 | (code & 0x3f)); + } + } + + /** + * A number as ASCII digits, written in place. Long.toString would allocate a + * String and its char[], and this is on the path of every response (the + * status and the content length) and every JSON number. + */ + public void putNumber(long value) { + if(value < 0) { + put('-'); + if(value == Long.MIN_VALUE) { + // Negating it overflows; it has no positive counterpart. + putAscii("9223372036854775808"); + return; + } + value = -value; + } + if(value == 0) { + put('0'); + return; + } + int digits = 0; + long counter = value; + while(counter > 0) { + digits++; + counter /= 10; + } + ensure(digits); + length += digits; + int at = length; + while(value > 0) { + data[--at] = (byte)('0' + (int)(value % 10)); + value /= 10; + } + } +} diff --git a/vm/backend/src/com/codename1/backend/Database.java b/vm/backend/src/com/codename1/backend/Database.java new file mode 100644 index 00000000000..c88a5e9bc8b --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Database.java @@ -0,0 +1,424 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.List; + +import com.codename1.backend.sql.MySql; +import com.codename1.backend.sql.Postgres; + +/** + * One database API over SQLite, PostgreSQL and MySQL, chosen by URL. + * + * 
+ *   Database.open("/var/lib/app/app.db")
+ *   Database.open(":memory:")
+ *   Database.open("postgres://user:secret@db.internal:5432/app?sslmode=require")
+ *   Database.open("mysql://user:secret@db.internal/app?sslmode=require"
+ *                 + "&sslrootcert=/etc/ssl/rds-ca.pem")
+ *
+ * + * The point of the single type is that a handler cannot tell which engine + * answered it. Rows come back as column-name to value maps whose values are + * always Long, Double, String, byte[] or null, whichever engine produced them -- + * so the same code developed against a local SQLite file runs against a managed + * PostgreSQL without a branch. Parameters are always bound, never interpolated, + * on all three. + * + * TLS has three settings, and none of them is "encrypted but unverified". + * `sslmode=require` demands TLS whose certificate chains to a trusted root AND + * carries the host's name; `sslmode=disable` is plaintext, deliberately; + * `sslmode=prefer` uses TLS when the server offers it and fails loudly rather + * than silently downgrading when verification does not hold. A managed instance + * or a development container that presents a private CA is reached by naming it: + * `sslrootcert=/path/to/ca.pem`. + * + * SQLite goes through {@link Db}, which is per-target: the engine is linked into + * the binary on the translated side and reached through a JDBC driver on the + * local Java SE side. PostgreSQL and MySQL are the SAME code on both targets -- + * they speak the wire protocol over {@link Tcp}, so there is no driver to install + * and nothing that can behave differently between the two. + * + * What differs between engines, and cannot be papered over: {@link #lastInsertId} + * is meaningful for SQLite and MySQL and always 0 for PostgreSQL, which has no + * such concept -- use `INSERT ... RETURNING id` there and read it as a row. + */ +public final class Database { + private final Db sqlite; + /** Db has no isClosed, so closure is recorded where it happens. */ + private boolean sqliteClosed; + private final Postgres postgres; + private final MySql mysql; + private final String describedAs; + + private Database(Db sqlite, Postgres postgres, MySql mysql, String describedAs) { + this.sqlite = sqlite; + this.postgres = postgres; + this.mysql = mysql; + this.describedAs = describedAs; + } + + /** A unit of work run inside {@link #transaction}. */ + public interface Work { + Object run(Database db) throws Exception; + } + + /** + * Opens the database the URL names. Anything that is not a recognised scheme + * is taken as a SQLite path, so an ordinary file name keeps working. + */ + public static Database open(String url) throws IOException { + if(url == null) { + throw new IOException("No database URL"); + } + if(url.startsWith("postgres://") || url.startsWith("postgresql://")) { + Url parsed = Url.parse(url, 5432); + return new Database(null, Postgres.connect(parsed.host, parsed.port, + parsed.path, parsed.user, parsed.password, parsed.sslMode, + parsed.caFile, parsed.timeoutMillis), null, parsed.describe("postgres")); + } + if(url.startsWith("mysql://") || url.startsWith("mariadb://")) { + Url parsed = Url.parse(url, 3306); + return new Database(null, null, MySql.connect(parsed.host, parsed.port, + parsed.path, parsed.user, parsed.password, parsed.sslMode, + parsed.caFile, parsed.timeoutMillis), parsed.describe("mysql")); + } + return new Database(Db.open(url), null, null, "sqlite:" + url); + } + + /** Wraps an already-open SQLite handle, for code that opened one directly. */ + public static Database of(Db db) { + return new Database(db, null, null, "sqlite"); + } + + /** Runs a statement that returns no rows. Returns the number of rows changed. */ + public int execute(String sql, Object[] params) throws IOException { + if(sqlite != null) { + return sqlite.execute(sql, params); + } + if(postgres != null) { + return postgres.execute(sql, params); + } + return mysql.execute(sql, params); + } + + /** Runs a query and returns every row as a column-name to value map. */ + public List query(String sql, Object[] params) throws IOException { + if(sqlite != null) { + return sqlite.query(sql, params); + } + if(postgres != null) { + return postgres.query(sql, params); + } + return mysql.query(sql, params); + } + + /** + * Runs body inside a transaction, committing when it returns and rolling back + * if it throws. + * + * SQLite gets BEGIN IMMEDIATE, which takes the write lock up front rather than + * discovering the conflict at the first write; the other two get a plain + * BEGIN, which is what they support. + */ + public Object transaction(Work body) throws Exception { + if(sqlite != null) { + final Work outer = body; + final Database self = this; + return sqlite.transaction(new Db.Work() { + public Object run(Db ignored) throws Exception { + return outer.run(self); + } + }); + } + control("BEGIN"); + boolean committed = false; + try { + Object result = body.run(this); + control("COMMIT"); + committed = true; + return result; + } finally { + if(!committed) { + try { + control("ROLLBACK"); + } catch (Exception err) { + // The original failure is the one worth reporting; a rollback + // that also fails must not replace it. + System.err.println("rollback failed: " + err); + } + } + } + } + + /** + * Transaction control. MySQL will not PREPARE these statements, so they go + * through its text protocol; PostgreSQL prepares them like anything else. The + * strings are constants in this file, never a caller's, which is what keeps + * the text path from being an injection route. + */ + private void control(String sql) throws IOException { + if(mysql != null) { + if("BEGIN".equals(sql)) { + mysql.begin(); + } else if("COMMIT".equals(sql)) { + mysql.commit(); + } else { + mysql.rollback(); + } + return; + } + execute(sql, null); + } + + /** + * The id the most recent insert produced, or 0 where the engine has no such + * concept. PostgreSQL is the case that has none: use INSERT ... RETURNING. + */ + public long lastInsertId() { + if(sqlite != null) { + return sqlite.lastInsertId(); + } + if(mysql != null) { + return mysql.lastInsertId(); + } + return 0; + } + + /** + * SQLite-only tuning, ignored elsewhere. Write-ahead logging is what lets + * readers run while a writer is active, and it has no counterpart on a server + * engine that already does. + */ + public void tuneForConcurrency(int busyTimeoutMillis) throws IOException { + if(sqlite != null) { + sqlite.enableWriteAheadLog(); + sqlite.setBusyTimeout(busyTimeoutMillis); + } + } + + /** The underlying SQLite handle, or null when this is a server engine. */ + public Db asSqlite() { + return sqlite; + } + + public void close() { + if(sqlite != null) { + sqliteClosed = true; + sqlite.close(); + } else if(postgres != null) { + postgres.close(); + } else { + mysql.close(); + } + } + + /** Whether this connection is still usable, which a pool has to know. */ + public boolean isOpen() { + if(postgres != null) { + return !postgres.isClosed(); + } + if(mysql != null) { + return !mysql.isClosed(); + } + // The SQLite arm used to answer "yes" whatever had happened to it, so a + // pool asking the documented usability question put a CLOSED connection + // back and the next caller got "Database is closed" instead. Db has no + // isClosed of its own, so closure is recorded here, where it happens. + return sqlite != null && !sqliteClosed; + } + + public String toString() { + return describedAs; + } + + /** + * The bit of URL parsing these two schemes need, written here rather than + * pulled from java.net: URI is not on the server-safe surface, and the + * password in the userinfo has to be percent-decoded, which a hand-rolled + * split usually forgets. + */ + private static final class Url { + String host = "localhost"; + int port; + String path = ""; + String user = ""; + String password = ""; + String sslMode = "prefer"; + String caFile; + int timeoutMillis = 10000; + + static Url parse(String url, int defaultPort) throws IOException { + Url out = new Url(); + out.port = defaultPort; + int schemeEnd = url.indexOf("://"); + String rest = url.substring(schemeEnd + 3); + String query = ""; + int queryAt = rest.indexOf('?'); + if(queryAt >= 0) { + query = rest.substring(queryAt + 1); + rest = rest.substring(0, queryAt); + } + int slash = rest.indexOf('/'); + if(slash >= 0) { + out.path = decode(rest.substring(slash + 1)); + rest = rest.substring(0, slash); + } + int at = rest.lastIndexOf('@'); + if(at >= 0) { + String credentials = rest.substring(0, at); + rest = rest.substring(at + 1); + int colon = credentials.indexOf(':'); + if(colon >= 0) { + out.user = decode(credentials.substring(0, colon)); + out.password = decode(credentials.substring(colon + 1)); + } else { + out.user = decode(credentials); + } + } + if(rest.length() > 0) { + int colon = rest.lastIndexOf(':'); + // A bare IPv6 literal has colons of its own; only a colon after the + // closing bracket is a port. + int bracket = rest.lastIndexOf(']'); + if(colon > bracket) { + out.host = strip(rest.substring(0, colon)); + try { + out.port = Integer.parseInt(rest.substring(colon + 1).trim()); + } catch (NumberFormatException err) { + throw new IOException("Not a port number in " + url); + } + } else { + out.host = strip(rest); + } + } + applyQuery(out, query); + return out; + } + + private static void applyQuery(Url out, String query) throws IOException { + int at = 0; + while(at < query.length()) { + int end = query.indexOf('&', at); + if(end < 0) { + end = query.length(); + } + String pair = query.substring(at, end); + at = end + 1; + int equals = pair.indexOf('='); + if(equals < 0) { + continue; + } + String key = pair.substring(0, equals); + String value = decode(pair.substring(equals + 1)); + if("sslmode".equals(key) || "ssl".equals(key)) { + if(!"require".equals(value) && !"prefer".equals(value) + && !"disable".equals(value)) { + throw new IOException("sslmode must be require, prefer or " + + "disable, not '" + value + "'"); + } + out.sslMode = value; + } else if("user".equals(key)) { + out.user = value; + } else if("password".equals(key)) { + out.password = value; + } else if("sslrootcert".equals(key) || "sslca".equals(key)) { + out.caFile = value; + } else if("connectTimeout".equals(key)) { + try { + out.timeoutMillis = Integer.parseInt(value.trim()); + } catch (NumberFormatException err) { + throw new IOException("connectTimeout must be a number of " + + "milliseconds, not '" + value + "'"); + } + // A negative one is refused here so the two arms cannot fail + // differently: Java SE throws IllegalArgumentException out of + // Socket.connect, while the packaged client reads any + // non-positive value as "block forever" and waits out the + // OS TCP timeout. Same URL, one an error and the other a + // hang, which is the worst kind of difference to debug. + if(out.timeoutMillis < 0) { + throw new IOException("connectTimeout must not be negative: '" + + value + "'. Use 0 for the platform default."); + } + } + } + } + + /** Never includes the password: this ends up in logs. */ + String describe(String scheme) { + return scheme + "://" + user + "@" + host + ":" + port + "/" + path + + " (sslmode=" + sslMode + + (caFile == null ? "" : ", sslrootcert=" + caFile) + ")"; + } + + private static String strip(String host) { + if(host.length() > 1 && host.charAt(0) == '[' + && host.charAt(host.length() - 1) == ']') { + return host.substring(1, host.length() - 1); + } + return host; + } + + private static String decode(String value) { + if(value.indexOf('%') < 0) { + return value; + } + byte[] out = new byte[value.length()]; + int length = 0; + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + if(c == '%' && iter + 2 < value.length()) { + int high = digit(value.charAt(iter + 1)); + int low = digit(value.charAt(iter + 2)); + if(high >= 0 && low >= 0) { + out[length++] = (byte)((high << 4) | low); + iter += 2; + continue; + } + } + out[length++] = (byte)c; + } + try { + return new String(out, 0, length, "UTF-8"); + } catch (UnsupportedEncodingException err) { + return value; + } + } + + private static int digit(char c) { + if(c >= '0' && c <= '9') { + return c - '0'; + } + if(c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if(c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + } +} diff --git a/vm/backend/src/com/codename1/backend/DbPool.java b/vm/backend/src/com/codename1/backend/DbPool.java new file mode 100644 index 00000000000..3d3790a34a4 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/DbPool.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * A fixed pool of connections to one SQLite database. + * + * Why a pool at all, when SQLite is compiled SQLITE_THREADSAFE=1 and a single + * connection is already safe to share: serialized mode makes concurrent use SAFE + * by serializing it, which means one connection gives no read concurrency at all. + * Several connections against a WAL database do, because WAL lets readers proceed + * while a writer is active. + * + * The pool is deliberately fixed-size and blocking rather than growing on demand: + * an unbounded pool against a single file just moves the contention into SQLite + * and makes the busy-timeout the thing that fails. + */ +public final class DbPool { + private final List idle = new ArrayList(); + private final List all = new ArrayList(); + private boolean closed; + + private DbPool() { + } + + /** + * Opens `size` connections to `path` and puts the database in WAL mode. + * + * A ":memory:" database cannot be pooled - each connection would get its OWN + * private database - so that is rejected rather than silently giving every + * caller a different empty database. + */ + public static DbPool open(String path, int size, int busyTimeoutMillis) throws IOException { + if(path == null || ":memory:".equals(path)) { + throw new IOException("An in-memory database cannot be pooled: each connection " + + "would get its own. Use Db.open(\":memory:\") directly."); + } + if(size < 1) { + throw new IOException("Pool size must be at least 1"); + } + DbPool pool = new DbPool(); + try { + for(int iter = 0 ; iter < size ; iter++) { + Db db = Db.open(path); + db.setBusyTimeout(busyTimeoutMillis); + if(iter == 0) { + // WAL is a property of the database file, not of the connection, + // so it only needs setting once - but every connection needs its + // own busy timeout. + db.enableWriteAheadLog(); + } + pool.all.add(db); + pool.idle.add(db); + } + } catch (IOException err) { + pool.close(); + throw err; + } + return pool; + } + + /** + * Takes a connection, blocking until one is free. Always release it in a + * finally, or prefer {@link #withConnection}, which cannot leak one. + */ + public synchronized Db borrow() throws IOException { + // Checked before the idle list, not only when it is empty. close() can run + // while a borrower still holds a connection, and that borrower's finally + // releases afterwards -- so the list can be non-empty after closing, and a + // check that only guards the empty case hands out a closed connection. + if(closed) { + throw new IOException("Pool is closed"); + } + while(idle.isEmpty()) { + if(closed) { + throw new IOException("Pool is closed"); + } + try { + wait(); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while waiting for a connection"); + } + } + return (Db)idle.remove(idle.size() - 1); + } + + public synchronized void release(Db db) { + if(db == null) { + return; + } + // A release that arrives after close() belongs to a borrower that was still + // running when the pool shut down. close() has already closed every + // connection, so putting this one back would repopulate an idle list nobody + // may draw from again. + if(closed) { + db.close(); + return; + } + idle.add(db); + notifyAll(); + } + + /** Borrows a connection, runs body, and returns it however body ends. */ + public Object withConnection(Db.Work body) throws Exception { + Db db = borrow(); + try { + return body.run(db); + } finally { + release(db); + } + } + + /** Convenience: one transaction on a pooled connection. */ + public Object inTransaction(final Db.Work body) throws Exception { + return withConnection(new Db.Work() { + public Object run(Db db) throws Exception { + return db.transaction(body); + } + }); + } + + public synchronized void close() { + closed = true; + for(int iter = 0 ; iter < all.size() ; iter++) { + ((Db)all.get(iter)).close(); + } + all.clear(); + idle.clear(); + notifyAll(); + } +} diff --git a/vm/backend/src/com/codename1/backend/Handler.java b/vm/backend/src/com/codename1/backend/Handler.java new file mode 100644 index 00000000000..93242cb4fe7 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Handler.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * What a Lambda-style handler implements. The event and the return value are raw + * JSON strings at this layer; the generated dispatcher from a @RestClient + * interface is what turns them into typed calls. + */ +public interface Handler { + /** + * - `event`: the invocation payload, as JSON + * - `requestId`: the host runtime's id for this invocation, for correlating logs + * + * Returns the response payload as JSON. Throwing is reported to the host + * runtime as an invocation error. + */ + String handle(String event, String requestId) throws Exception; +} diff --git a/vm/backend/src/com/codename1/backend/Http.java b/vm/backend/src/com/codename1/backend/Http.java new file mode 100644 index 00000000000..7491751b93f --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Http.java @@ -0,0 +1,356 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * A minimal HTTP/1.1 client over [Tcp]. Enough to speak a host runtime's control + * protocol - the AWS Lambda Runtime API, for instance - without pulling in a + * platform layer or a TLS stack. Plaintext only: the Lambda Runtime API is + * plaintext on the loopback interface, and anything facing the public internet + * terminates TLS in front of the process. + */ +public final class Http { + private Http() { + } + + /** One HTTP response: status line, headers and body. */ + public static final class Response { + private final int status; + private final List headerNames; + private final List headerValues; + private final byte[] body; + + Response(int status, List headerNames, List headerValues, byte[] body) { + this.status = status; + this.headerNames = headerNames; + this.headerValues = headerValues; + this.body = body; + } + + public int getStatus() { + return status; + } + + public byte[] getBody() { + return body; + } + + public String getBodyAsString() { + try { + return new String(body, "UTF-8"); + } catch (IOException err) { + return new String(body); + } + } + + /** Case-insensitive, because header case is not guaranteed by anything. */ + public String getHeader(String name) { + for(int iter = 0 ; iter < headerNames.size() ; iter++) { + if(((String)headerNames.get(iter)).equalsIgnoreCase(name)) { + return (String)headerValues.get(iter); + } + } + return null; + } + } + + public static Response get(String host, int port, String path) throws IOException { + return request(host, port, "GET", path, null); + } + + public static Response post(String host, int port, String path, byte[] body) throws IOException { + return request(host, port, "POST", path, body); + } + + public static Response request(String host, int port, String method, String path, byte[] body) throws IOException { + Tcp socket = Tcp.connect(host, port, 0); + try { + StringBuilder head = new StringBuilder(); + head.append(method).append(' ').append(path).append(" HTTP/1.1\r\n"); + head.append("Host: ").append(host).append(':').append(port).append("\r\n"); + head.append("Connection: close\r\n"); + head.append("Content-Length: ").append(body == null ? 0 : body.length).append("\r\n"); + head.append("\r\n"); + byte[] headBytes = head.toString().getBytes("UTF-8"); + socket.write(headBytes, 0, headBytes.length); + if(body != null && body.length > 0) { + socket.write(body, 0, body.length); + } + return readResponse(socket); + } finally { + socket.close(); + } + } + + private static Response readResponse(Tcp socket) throws IOException { + // "Connection: close" is requested above, so the whole response can be read + // to end-of-stream and parsed in memory. That keeps the parser free of the + // chunked/keep-alive state machine, at the cost of one connection per call -- + // which on loopback is cheaper than the code it saves. + ByteArrayOutputStream raw = new ByteArrayOutputStream(); + byte[] chunk = new byte[4096]; + while(true) { + int n = socket.read(chunk, 0, chunk.length); + if(n <= 0) { + break; + } + raw.write(chunk, 0, n); + } + byte[] all = raw.toByteArray(); + if(all.length == 0) { + // The peer closed without sending anything. Reporting this as malformed + // HTTP sent every "the host went away" shutdown to the wrong diagnosis. + throw new IOException("Connection closed before any response was sent"); + } + int headerEnd = indexOfHeaderEnd(all); + if(headerEnd < 0) { + throw new IOException("Malformed HTTP response: no header terminator"); + } + String headerText = new String(all, 0, headerEnd, "UTF-8"); + String[] lines = split(headerText, "\r\n"); + if(lines.length == 0) { + throw new IOException("Malformed HTTP response: empty"); + } + int status = parseStatus(lines[0]); + List names = new ArrayList(); + List values = new ArrayList(); + for(int iter = 1 ; iter < lines.length ; iter++) { + int colon = lines[iter].indexOf(':'); + if(colon > 0) { + names.add(lines[iter].substring(0, colon).trim()); + values.add(lines[iter].substring(colon + 1).trim()); + } + } + int bodyStart = headerEnd + 4; + byte[] bodyBytes = new byte[all.length - bodyStart]; + System.arraycopy(all, bodyStart, bodyBytes, 0, bodyBytes.length); + // Everything before EOF is not the same as the whole body. A connection + // that dies mid-payload leaves a SHORT one, and handing that back as a + // complete response is how a Lambda handler is invoked on half an event + // and produces side effects from input the caller never sent. The + // declared length is the peer's own statement of what it owed, so a + // shortfall is a transport failure and is reported as one. + int declared = declaredLength(names, values); + if(declared >= 0 && bodyBytes.length < declared) { + throw new IOException("The response body stopped after " + bodyBytes.length + + " of the " + declared + " byte(s) its Content-Length declared, so " + + "the connection failed part way through it"); + } + return new Response(status, names, values, decodeBody(bodyBytes, names, values)); + } + + /** + * The Content-Length the peer declared, or -1 when it declared none or the + * value is not a number. A chunked response has no Content-Length, so the + * check above simply does not apply to one. + */ + private static int declaredLength(List names, List values) { + for(int iter = 0 ; iter < names.size() ; iter++) { + if("content-length".equalsIgnoreCase(String.valueOf(names.get(iter)))) { + try { + return Integer.parseInt(String.valueOf(values.get(iter)).trim()); + } catch (NumberFormatException err) { + return -1; + } + } + } + return -1; + } + + /** + * Strips whatever Transfer-Encoding the peer applied, which is usually none. + * + * Reading to EOF is not the same as reading the body: `Connection: close` ends + * the message but does not remove chunk framing, so a server that answers + * chunked hands back size lines and terminators mixed into the payload. The + * Lambda Runtime API sends Content-Length today, which is the reason to handle + * this rather than a reason not to -- nothing here fails until the day it does. + * + * A coding this client cannot undo is an error rather than a pass-through. The + * one outcome worth ruling out is returning framed bytes as though they were + * the body, because the handler then parses garbage and blames its own input. + */ + private static byte[] decodeBody(byte[] body, List names, List values) throws IOException { + String encoding = joinedHeader(names, values, "Transfer-Encoding"); + if(encoding == null) { + return body; + } + String[] codings = split(encoding, ","); + boolean chunked = false; + for(int iter = 0 ; iter < codings.length ; iter++) { + String coding = codings[iter].trim(); + if(coding.length() == 0 || coding.equalsIgnoreCase("identity")) { + continue; + } + // Case folding a protocol token with toLowerCase() is locale sensitive and + // wrong on a Turkish device; equalsIgnoreCase compares character by + // character and is not. + if(coding.equalsIgnoreCase("chunked") && iter == codings.length - 1) { + chunked = true; + continue; + } + throw new IOException("Unsupported Transfer-Encoding: " + encoding); + } + return chunked ? dechunk(body) : body; + } + + /** + * Every value sent under one header name, joined the way a single line would + * have read. A field may legally arrive split across repeated lines. + */ + private static String joinedHeader(List names, List values, String name) { + StringBuilder joined = null; + for(int iter = 0 ; iter < names.size() ; iter++) { + if(((String)names.get(iter)).equalsIgnoreCase(name)) { + if(joined == null) { + joined = new StringBuilder(); + } else { + joined.append(','); + } + joined.append((String)values.get(iter)); + } + } + return joined == null ? null : joined.toString(); + } + + /** Reassembles a chunked body, dropping the framing and any trailer section. */ + private static byte[] dechunk(byte[] data) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int pos = 0; + while(true) { + int eol = indexOfCrLf(data, pos); + if(eol < 0) { + throw new IOException("Truncated chunked response: no chunk size line"); + } + int end = pos; + while(end < eol && data[end] != ';') { + end++; + } + int size = parseChunkSize(data, pos, end); + pos = eol + 2; + if(size == 0) { + // Trailers may follow. They are header fields, not body bytes, and + // this client has no caller that reads them. + return out.toByteArray(); + } + if(size > data.length - pos) { + throw new IOException("Truncated chunked response: chunk runs past the body"); + } + out.write(data, pos, size); + pos += size; + if(pos + 2 > data.length || data[pos] != '\r' || data[pos + 1] != '\n') { + throw new IOException("Malformed chunked response: chunk not terminated"); + } + pos += 2; + } + } + + private static int indexOfCrLf(byte[] data, int from) { + for(int iter = from ; iter + 1 < data.length ; iter++) { + if(data[iter] == '\r' && data[iter + 1] == '\n') { + return iter; + } + } + return -1; + } + + /** + * A chunk size is hexadecimal and unsigned. Parsed by hand because the sizes + * this guards against are exactly the ones that overflow a signed parse. + */ + private static int parseChunkSize(byte[] data, int from, int to) throws IOException { + int size = 0; + int digits = 0; + for(int iter = from ; iter < to ; iter++) { + int c = data[iter] & 0xff; + int digit; + if(c >= '0' && c <= '9') { + digit = c - '0'; + } else if(c >= 'a' && c <= 'f') { + digit = c - 'a' + 10; + } else if(c >= 'A' && c <= 'F') { + digit = c - 'A' + 10; + } else if((c == ' ' || c == '\t') && digits > 0) { + break; + } else { + throw new IOException("Malformed chunk size"); + } + if(size > (Integer.MAX_VALUE - digit) / 16) { + throw new IOException("Chunk size out of range"); + } + size = size * 16 + digit; + digits++; + } + if(digits == 0) { + throw new IOException("Malformed chunk size: empty"); + } + return size; + } + + private static int indexOfHeaderEnd(byte[] data) { + for(int iter = 0 ; iter + 3 < data.length ; iter++) { + if(data[iter] == '\r' && data[iter + 1] == '\n' && data[iter + 2] == '\r' && data[iter + 3] == '\n') { + return iter; + } + } + return -1; + } + + private static int parseStatus(String statusLine) throws IOException { + int first = statusLine.indexOf(' '); + if(first < 0) { + throw new IOException("Malformed status line: " + statusLine); + } + int second = statusLine.indexOf(' ', first + 1); + String code = second < 0 ? statusLine.substring(first + 1) : statusLine.substring(first + 1, second); + try { + return Integer.parseInt(code.trim()); + } catch (NumberFormatException err) { + throw new IOException("Malformed status code: " + statusLine); + } + } + + private static String[] split(String value, String separator) { + List parts = new ArrayList(); + int pos = 0; + while(true) { + int next = value.indexOf(separator, pos); + if(next < 0) { + parts.add(value.substring(pos)); + break; + } + parts.add(value.substring(pos, next)); + pos = next + separator.length(); + } + String[] result = new String[parts.size()]; + for(int iter = 0 ; iter < result.length ; iter++) { + result[iter] = (String)parts.get(iter); + } + return result; + } +} diff --git a/vm/backend/src/com/codename1/backend/Http1Date.java b/vm/backend/src/com/codename1/backend/Http1Date.java new file mode 100644 index 00000000000..5f35afa23c6 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Http1Date.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * The one date format HTTP/1.1 requires on the wire ("Sun, 06 Nov 1994 08:49:37 + * GMT"), formatted and parsed from epoch milliseconds directly. + * + * Done arithmetically rather than through Calendar and TimeZone: the format is + * fixed and always GMT, and going through a calendar would make a header depend on + * the process's default time zone, which is how Last-Modified ends up hours off on + * a machine that is not in UTC. + */ +public final class Http1Date { + private static final String[] DAYS = {"Thu", "Fri", "Sat", "Sun", "Mon", "Tue", "Wed"}; + private static final String[] MONTHS = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + + private Http1Date() { + } + + // JavaAPI's Math has no floorDiv/floorMod, and integer division in Java + // truncates toward zero -- which for a pre-1970 timestamp gives the wrong day. + private static long floorDiv(long x, long y) { + long q = x / y; + if((x % y != 0) && ((x < 0) != (y < 0))) { + q--; + } + return q; + } + + private static long floorMod(long x, long y) { + return x - floorDiv(x, y) * y; + } + + public static String format(long millis) { + long seconds = floorDiv(millis, 1000L); + long days = floorDiv(seconds, 86400L); + int secondOfDay = (int)floorMod(seconds, 86400L); + // 1970-01-01 was a Thursday, which is why DAYS starts there. + int dayOfWeek = (int)floorMod(days, 7L); + int[] civil = civilFromDays(days); + StringBuilder out = new StringBuilder(29); + out.append(DAYS[dayOfWeek]).append(", "); + two(out, civil[2]).append(' ').append(MONTHS[civil[1] - 1]).append(' '); + out.append(civil[0]).append(' '); + two(out, secondOfDay / 3600).append(':'); + two(out, (secondOfDay / 60) % 60).append(':'); + two(out, secondOfDay % 60).append(" GMT"); + return out.toString(); + } + + /** Epoch millis, or -1 when the value is not a date this understands. */ + public static long parse(String value) { + if(value == null) { + return -1; + } + String v = value.trim(); + // "Sun, 06 Nov 1994 08:49:37 GMT" -- the only form a modern server must + // emit. The two obsolete RFC 850 / asctime forms are not accepted; a client + // sending one gets a full response rather than a wrong 304. + // The WHOLE shape, not the length and one comma. Every field below is + // read by fixed offset and then handed to daysFromCivil, which NORMALISES + // whatever it is given: "Sun, 99 Nov 9999 99:99:99 BAD" was accepted and + // turned into a date far in the future, and StaticFiles then read that as + // "newer than the file" and answered 304 -- a conditional request served + // no content because its date was nonsense. A malformed date has to be + // no date at all. + if(v.length() != 29 || v.charAt(3) != ',' || v.charAt(4) != ' ' + || v.charAt(7) != ' ' || v.charAt(11) != ' ' || v.charAt(16) != ' ' + || v.charAt(19) != ':' || v.charAt(22) != ':' || v.charAt(25) != ' ' + || !"GMT".equals(v.substring(26))) { + return -1; + } + try { + int day = Integer.parseInt(v.substring(5, 7).trim()); + String monthName = v.substring(8, 11); + int month = -1; + for(int iter = 0 ; iter < MONTHS.length ; iter++) { + if(MONTHS[iter].equals(monthName)) { + month = iter + 1; + break; + } + } + if(month < 0) { + return -1; + } + int year = Integer.parseInt(v.substring(12, 16).trim()); + int hour = Integer.parseInt(v.substring(17, 19).trim()); + int minute = Integer.parseInt(v.substring(20, 22).trim()); + int second = Integer.parseInt(v.substring(23, 25).trim()); + // Ranges, for the same reason: daysFromCivil answers for day 99 as + // readily as for day 9, and the answer is a different date than the + // one written. A second of 60 is allowed because a leap second is + // spelled that way. + if(day < 1 || day > daysInMonth(year, month) || hour > 23 || minute > 59 + || second > 60 || year < 1) { + return -1; + } + long days = daysFromCivil(year, month, day); + return ((days * 86400L) + hour * 3600L + minute * 60L + second) * 1000L; + } catch (NumberFormatException err) { + return -1; + } catch (IndexOutOfBoundsException err) { + return -1; + } + } + + /** Days in a month, so a date that does not exist is not silently moved. */ + private static int daysInMonth(int year, int month) { + switch(month) { + case 2: + boolean leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; + return leap ? 29 : 28; + case 4: case 6: case 9: case 11: + return 30; + default: + return 31; + } + } + + private static StringBuilder two(StringBuilder out, int value) { + if(value < 10) { + out.append('0'); + } + return out.append(value); + } + + /* + * Howard Hinnant's civil-date algorithms: exact for every date in range, with + * no leap-year special cases to get wrong. The shift moves the epoch to + * 0000-03-01 so February -- the only month whose length varies -- lands at the + * end of the year and drops out of the arithmetic. + */ + private static int[] civilFromDays(long days) { + long z = days + 719468L; + long era = floorDiv(z, 146097L); + long doe = z - era * 146097L; + long yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + long y = yoe + era * 400L; + long doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + long mp = (5 * doy + 2) / 153; + long d = doy - (153 * mp + 2) / 5 + 1; + long m = mp < 10 ? mp + 3 : mp - 9; + return new int[]{(int)(m <= 2 ? y + 1 : y), (int)m, (int)d}; + } + + private static long daysFromCivil(int year, int month, int day) { + long y = year - (month <= 2 ? 1 : 0); + long era = floorDiv(y, 400L); + long yoe = y - era * 400L; + long mp = month > 2 ? month - 3 : month + 9; + long doy = (153 * mp + 2) / 5 + day - 1; + long doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097L + doe - 719468L; + } +} diff --git a/vm/backend/src/com/codename1/backend/HttpServer.java b/vm/backend/src/com/codename1/backend/HttpServer.java new file mode 100644 index 00000000000..c8ca644c8c9 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/HttpServer.java @@ -0,0 +1,5359 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * An HTTP/1.1 server built around a reactor and a bounded worker pool. + * + * The split is the whole design. A parked ParparVM thread costs about 243KB on + * musl (see vm/benchmarks ThreadCost), so ten thousand connections cannot each + * have one. Here an IDLE connection is a descriptor the reactor watches, and only + * a connection with a request in flight occupies a worker. Workers are bounded, so + * overload sheds as queueing rather than as memory exhaustion. + * + * Once a worker owns a connection its descriptor is switched to BLOCKING mode, so + * request parsing is a straight read loop rather than a resumable state machine. + * That spends a worker per in-flight request to avoid a large amount of + * complexity, and in-flight requests are the thing there are few of. + */ +public final class HttpServer { + /** + * What a handler receives. Header names are matched case-insensitively. + * + * The headers are NOT copied out of the request. They stay as offsets into + * the buffer the kernel filled, and {@link #getHeader} compares against those + * bytes -- so a handler that reads two headers allocates nothing, where + * building a map of Strings cost about 2.6KB per request and made char[] the + * single largest allocation in the server. {@link #getHeaders} still returns + * a Map, built on first call, for callers that want one. + * + * A Request is valid for the duration of {@link Handler#handle} and NOT + * beyond it. Both the byte array it was parsed from and the slice table that + * indexes it are reused -- the array by the reading thread, the table by the + * connection -- so a Request held past the handler describes whatever arrived + * next, not what it was built from. + * + * This corrects a claim that used to stand here, that the slices "stay valid + * for as long as the Request is held". That was never true: the table is + * `conn.slices`, reused by the very next request on the same connection. The + * zero-copy read added a second way for it to be false, which is what prompted + * reading the sentence carefully enough to notice it had always been wrong. + * + * The synchronous handler signature already makes the call the natural + * lifetime, so this documents the contract rather than narrowing one -- but + * anything that needs to outlive the handler must copy what it needs, and + * {@link #getHeaders} or {@link #getHeader} give Strings that are safe to keep. + */ + public static final class Request { + // Not final because one Request is REUSED for every request on a + // connection, which is the same trade the buffer and the slice table + // already make and the reason a Request is documented as valid only for + // the duration of Handler.handle. "Immutable to its handler" is the + // property that matters and it still holds exactly: reset() runs while a + // request is being PARSED, which is strictly before the handler is called + // and strictly after the previous one returned. + private String method; + private String target; + private String version; + private String body; + /** The connection this request arrived on; null for HTTP/2, see respond. */ + private Conn conn; + /** The bytes the header block was read from. */ + private byte[] raw; + /** nameStart, nameLength, valueStart, valueLength per header, in order. */ + private int[] slices; + private int headerCount; + private Map headers; + /** + * Where the request target sits inside {@link #raw}. + * + * Kept so a router can match on the bytes the parser already has. Matching + * on getTarget() means comparing Strings, and the generated router is the + * one caller that runs for every request on every route, so it is worth not + * asking it to. + */ + private int targetStart; + private int targetLength; + /** Computed on first use; -1 until then. Reset with the rest of the Request. */ + private int pathLength = -1; + + Request(String method, String target, String version, byte[] raw, int[] slices, + int headerCount, String body) { + this(method, target, version, raw, slices, headerCount, body, 0, 0); + } + + Request(String method, String target, String version, byte[] raw, int[] slices, + int headerCount, String body, int targetStart, int targetLength) { + this.method = method; + this.target = target; + this.version = version; + this.raw = raw; + this.slices = slices; + this.headerCount = headerCount; + this.body = body; + this.targetStart = targetStart; + this.targetLength = targetLength; + this.pathLength = -1; + } + + /** + * True when the request PATH is exactly these bytes. + * + * The path, not the target: everything from `?` onwards is the query string + * and is not part of the route. A router that compared the whole target + * would match `/healthz` and miss `/healthz?probe=1`, which is the same + * request. + * + * For the generated router, which holds each route as a byte[] constant. No + * String is built and nothing is hashed: it is a length test and a compare + * against the buffer the request was parsed from. Falls back to comparing + * the target String when the slice is not available, which is the HTTP/2 + * path -- there the target came from HPACK rather than from a byte range. + */ + public boolean pathIs(byte[] path) { + if(path == null) { + return false; + } + int length = pathByteLength(); + if(path.length != length) { + return false; + } + return regionEquals(path, 0, length); + } + + /** As {@link #pathIs}, for a route that continues into a path variable. */ + public boolean pathStartsWith(byte[] prefix) { + if(prefix == null || prefix.length > pathByteLength()) { + return false; + } + return regionEquals(prefix, 0, prefix.length); + } + + /** + * The path from `from` onwards, as text. Allocates, so a matched route only. + * + * Percent escapes are left alone. The router decodes the segments it binds, + * because decoding first would let an encoded `/` invent a segment boundary + * that the client never sent. + */ + public String pathFrom(int from) { + int length = pathByteLength(); + if(from >= length) { + return ""; + } + if(targetLength <= 0 || raw == null) { + return target.substring(from, length); + } + return asciiString(raw, targetStart + from, length - from); + } + + /** The path's length in bytes -- the target up to `?` -- without building it. */ + public int pathByteLength() { + if(pathLength >= 0) { + return pathLength; + } + int length = targetLength > 0 ? targetLength + : (target == null ? 0 : target.length()); + int found = length; + for(int iter = 0 ; iter < length ; iter++) { + if(byteAt(iter) == '?') { + found = iter; + break; + } + } + pathLength = found; + return found; + } + + /** + * A query parameter's decoded value, or null when the request did not send + * it. An empty `?flag=` is present with an empty value, which is not the + * same as absent, and callers that offer a default depend on the difference. + */ + public String queryParam(String name) { + int length = targetLength > 0 ? targetLength + : (target == null ? 0 : target.length()); + int pos = pathByteLength(); + if(pos >= length || name == null) { + return null; + } + pos++; // the '?' itself + while(pos <= length) { + int end = pos; + while(end < length && byteAt(end) != '&') { + end++; + } + int eq = pos; + while(eq < end && byteAt(eq) != '=') { + eq++; + } + if(nameEquals(name, pos, eq)) { + return percentDecode(eq < end ? eq + 1 : end, end); + } + pos = end + 1; + } + return null; + } + + /** One byte of the request target, from whichever form this Request holds. */ + private int byteAt(int index) { + if(targetLength > 0 && raw != null) { + return raw[targetStart + index] & 0xff; + } + return target.charAt(index) & 0xff; + } + + private boolean regionEquals(byte[] expected, int from, int length) { + for(int iter = 0 ; iter < length ; iter++) { + if(byteAt(from + iter) != (expected[iter] & 0xff)) { + return false; + } + } + return true; + } + + /** + * Compares a parameter name against the raw bytes, decoding escapes in the + * request as it goes. Names are rarely encoded, but comparing an encoded + * name against a plain one would silently miss the parameter. + */ + private boolean nameEquals(String name, int from, int to) { + // A DECODED OCTET is compared below, so the thing it is compared + // against has to be an octet too. For a non-ASCII name it is not: + // "cafe" with an acute e arrives as caf%C3%A9, whose octets are 0xC3 + // 0xA9, while the Java char is 0xE9 -- no octet ever equals it, and + // the parameter reads as absent even though the client sent it + // exactly as every client encodes it. Such a name is compared + // against its UTF-8 bytes instead. ASCII names, which is nearly all + // of them, keep the character path: it is identical for them and + // allocates nothing on a per-request code path. + byte[] utf8 = null; + for(int iter = 0 ; iter < name.length() ; iter++) { + if(name.charAt(iter) > 0x7f) { + try { + utf8 = name.getBytes("UTF-8"); + } catch (IOException err) { + return false; // it cannot be encoded, so it cannot match + } + break; + } + } + int wanted = utf8 == null ? name.length() : utf8.length; + int index = 0; + int pos = from; + while(pos < to) { + int c = byteAt(pos); + int width = 1; + if(c == '%' && pos + 2 < to) { + int hi = hexDigit(byteAt(pos + 1)); + int lo = hexDigit(byteAt(pos + 2)); + if(hi >= 0 && lo >= 0) { + c = (hi << 4) | lo; + width = 3; + } + } else if(c == '+') { + c = ' '; + } + int want = index >= wanted ? -1 + : (utf8 == null ? (name.charAt(index) & 0xff) : (utf8[index] & 0xff)); + if(want != c) { + return false; + } + index++; + pos += width; + } + return index == wanted; + } + + /** + * Decodes one query value. The octets are gathered and decoded as a run, + * because a percent escape carries one byte of UTF-8 and a character built + * from a single byte at a time is mojibake for everything above ASCII. + */ + private String percentDecode(int from, int to) { + byte[] out = new byte[to - from]; + int length = 0; + int pos = from; + while(pos < to) { + int c = byteAt(pos); + if(c == '%' && pos + 2 < to) { + int hi = hexDigit(byteAt(pos + 1)); + int lo = hexDigit(byteAt(pos + 2)); + if(hi >= 0 && lo >= 0) { + out[length++] = (byte)((hi << 4) | lo); + pos += 3; + continue; + } + } else if(c == '+') { + c = ' '; + } + out[length++] = (byte)c; + pos++; + } + try { + return new String(out, 0, length, "UTF-8"); + } catch (java.io.UnsupportedEncodingException err) { + // UTF-8 is required of every VM this runs on; the checked exception + // is the API's, not a case that can happen. + return new String(out, 0, length); + } + } + + private static int hexDigit(int c) { + if(c >= '0' && c <= '9') { + return c - '0'; + } + if(c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if(c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + + /** + * A Response for this request WITHOUT allocating one. + * + * Returns the connection's single Response, re-pointed to these values. It + * is valid for the duration of {@link Handler#handle} and not beyond it -- + * the same contract this Request already carries, and for the same reason: + * the next request on this connection reuses it. + * + * Why it exists: on a route that allocates nothing else, the Response was + * the last per-request allocation, and allocation is what drives both the + * collector's frequency and its footprint. Removing it measured a 15x + * better p99 and an 8x smaller resident set at the same throughput. + * + * {@code new Response(...)} still works and still allocates; a handler that + * needs its Response to outlive the call must use it. + */ + public Response respond(int status, String contentType, byte[] body) { + if(conn == null) { + return new Response(status, contentType, body); // HTTP/2 path + } + if(conn.pooledResponse == null) { + conn.pooledResponse = new Response(status, contentType, body); + } else { + conn.pooledResponse.reset(status, contentType, + body == null ? EMPTY_BODY : body, -1, 0, 0, null); + } + return conn.pooledResponse; + } + + /** + * A JSON response on the connection's pooled Response, serialised straight + * from the value. + * + * The deferred-JSON path already avoided every copy on the body side -- + * Json.write goes into the connection's reusable ByteSink, so nothing + * materialises a byte[] or a String -- but Response.jsonValue is a static + * that allocates a fresh Response per call, and that was the ONLY thing the + * route allocated. Profiled over 10.8M requests: 88.1 bytes each, all of it + * one HttpServer.Response, count 10789601 against 10789541 requests. The + * plaintext route had already been pooled and sat at 0.1 bytes per request. + * + * That is worth removing because of what allocation costs HERE rather than + * what it costs to allocate: the collector shares the server's cores, so a + * route that allocates pays for cycles in its tail. fasthttp on the same + * body allocates about 16 bytes per request and collects three times a + * second; this route was collecting thirteen to eighteen times a second. + * + * reset() clears deferredJson and hasDeferredJson, so a pooled Response + * reused for a plain body cannot carry a stale value into the next + * response -- which is the failure this would otherwise invite. + */ + public Response respondJson(int status, Object value) { + if(conn == null) { + return Response.jsonValue(status, value); // HTTP/2 path, as respond() does + } + if(conn.pooledResponse == null) { + conn.pooledResponse = new Response(status, JSON_CONTENT_TYPE, + EMPTY_BODY, -1, 0, 0, null); + } else { + conn.pooledResponse.reset(status, JSON_CONTENT_TYPE, + EMPTY_BODY, -1, 0, 0, null); + } + conn.pooledResponse.deferredJson = value; + conn.pooledResponse.hasDeferredJson = true; + return conn.pooledResponse; + } + + /** + * DIAGNOSTIC: the connection's Response exactly as the last request left + * it, or null the first time. Separates the allocation pooling saves from + * the field writes it adds -- see the bench demo's RESPONSE_MODE. + */ + public Response presetResponse() { + return conn == null ? null : conn.pooledResponse; + } + + /** + * Re-points this Request at a freshly parsed request. Every field is + * assigned, with no "unchanged" case: a field left behind describes the + * PREVIOUS request on this connection, and headers is the one that would + * hurt -- it caches a Map built on demand by getHeaders, so carrying it + * over would answer one request's header lookups with another's. That is + * a wrong answer rather than a crash, which is why it is assigned here + * unconditionally instead of being cleared at some later point. + */ + void reset(Conn conn, String method, String target, String version, byte[] raw, + int[] slices, int headerCount, String body) { + reset(conn, method, target, version, raw, slices, headerCount, body, 0, 0); + } + + void reset(Conn conn, String method, String target, String version, byte[] raw, + int[] slices, int headerCount, String body, + int targetStart, int targetLength) { + this.conn = conn; + this.method = method; + this.target = target; + this.version = version; + this.raw = raw; + this.slices = slices; + this.headerCount = headerCount; + this.body = body; + this.headers = null; + this.targetStart = targetStart; + this.targetLength = targetLength; + // Recomputed for this request. A stale value would give the next request + // on this connection the previous one's path length. + this.pathLength = -1; + } + + /** + * For HTTP/2, whose headers arrive already decoded from the HPACK state -- + * there is no request buffer to slice into, so the map IS the + * representation and every lookup below falls back to it. + */ + Request(String method, String target, String version, Map headers, String body) { + this.method = method; + this.target = target; + this.version = version; + this.raw = null; + this.slices = null; + this.headerCount = 0; + this.headers = headers; + this.body = body; + } + + /** "HTTP/1.1" or "HTTP/1.0". The two differ on whether keep-alive is the default. */ + public String getVersion() { + return version; + } + + public String getMethod() { + return method; + } + + /** Path plus query string, exactly as it arrived. */ + public String getTarget() { + return target; + } + + /** + * The headers as a Map, lower-cased names to values. + * + * Built on the first call and cached. Prefer {@link #getHeader}: this + * allocates a String per name and per value, which is the cost the slice + * representation exists to avoid. + */ + public Map getHeaders() { + if(headers == null) { + Map out = new LinkedHashMap(); + for(int iter = 0 ; iter < headerCount ; iter++) { + int base = iter * 4; + String name = lowerCaseString(raw, slices[base], slices[base + 1]); + String value = asciiString(raw, slices[base + 2], slices[base + 3]); + Object existing = out.get(name); + if(existing == null) { + out.put(name, value); + } else { + // Combined in arrival order, as the HTTP/2 path does. Replacing + // meant getHeader() answered with the FIRST occurrence while + // this map held the last, so a cookie split across two fields + // was visible through one API and gone from the other -- and a + // generated dispatcher reads this map. + out.put(name, String.valueOf(existing) + + ("cookie".equals(name) ? "; " : ",") + value); + } + } + headers = out; + } + return headers; + } + + /** One header by name, matched case-insensitively. Allocates only the value. */ + public String getHeader(String name) { + if(name == null) { + return null; + } + if(raw == null) { + Object v = headers.get(asciiLower(name)); + return v == null ? null : String.valueOf(v); + } + int at = indexOfHeader(name); + if(at < 0) { + return null; + } + if(countHeader(name) == 1) { + return asciiString(raw, slices[at + 2], slices[at + 3]); + } + // Repeated field. getHeaders() combines these and the HTTP/2 path does + // too; returning only the first meant a handler reading getHeader saw + // less than one reading getHeaders, and cookies split across two Cookie + // fields -- which is legal on the wire -- simply vanished from the + // second one. Authentication that reads a cookie could then differ by + // which API it used, or by protocol. + // + // Cookie joins with "; " because that is its own delimiter (RFC 6265); + // everything else with "," as RFC 9110 5.3 defines for a list field. + String separator = "cookie".equalsIgnoreCase(name) ? "; " : ", "; + StringBuilder joined = new StringBuilder(); + for(int iter = 0 ; iter < headerCount ; iter++) { + int base = iter * 4; + if(!sliceEqualsIgnoreCase(raw, slices[base], slices[base + 1], name)) { + continue; + } + if(joined.length() > 0) { + joined.append(separator); + } + joined.append(asciiString(raw, slices[base + 2], slices[base + 3])); + } + return joined.toString(); + } + + /** + * Lowercases an ASCII header name. + * + * NOT String.toLowerCase(), which is locale sensitive and has no overload + * here that takes a Locale: on a Turkish default the I of "COOKIE" folds to + * a dotless i and the lookup misses a header that is present. A field name + * is ASCII by specification, so it folds by hand. Six lines, copied rather + * than shared, as the other folds in this tree are. + */ + private static String asciiLower(String name) { + int length = name.length(); + StringBuilder out = new StringBuilder(length); + for(int iter = 0 ; iter < length ; iter++) { + char c = name.charAt(iter); + out.append(c >= 'A' && c <= 'Z' ? (char)(c + 32) : c); + } + return out.toString(); + } + + /** The slice index of a header, or -1. No allocation on either path. */ + int indexOfHeader(String name) { + // Callers guard on raw != null; headerCount is 0 for the map form, so + // this returns -1 there rather than reading a null slices array. + // + // The name is folded ONCE, not once per header. sliceEqualsIgnoreCase + // reads it with charAt and case-folds it on every comparison, so a + // four-header request folded the same needle four times over -- and the + // generated code shows why that is not free: each character costs a + // cn1InlStrCharAt (which re-checks the string's coder) plus a foldAscii + // call, against a plain array read on the other side. + byte[] needle = foldedBytes(name); + if(needle == null) { + // Not ASCII-foldable, so the general path is the only correct one. + for(int iter = 0 ; iter < headerCount ; iter++) { + int base = iter * 4; + if(sliceEqualsIgnoreCase(raw, slices[base], slices[base + 1], name)) { + return base; + } + } + return -1; + } + for(int iter = 0 ; iter < headerCount ; iter++) { + int base = iter * 4; + if(sliceEqualsFolded(raw, slices[base], slices[base + 1], needle)) { + return base; + } + } + return -1; + } + + /** + * Whether a header's value contains a token, case-insensitively. Used for + * "connection: keep-alive" and friends without materialising the value. + */ + /** + * Whether a comma-separated field lists this token. + * + * A WHOLE token, not a substring. "Connection: disclose" contains "close" + * and "not-keep-alive" contains "keep-alive", and a substring test read + * both as the option itself -- so an extension token nobody here has heard + * of decided whether the connection stays open, which is a framing + * decision made on an unrelated name. Every occurrence of the field is + * searched, because a repeated one is as legal as a repeated Cookie. + */ + boolean headerContains(String name, String token) { + if(raw == null) { + Object v = headers == null ? null : headers.get(asciiLower(name)); + return v != null && listHasToken(String.valueOf(v), token); + } + for(int iter = 0 ; iter < headerCount ; iter++) { + int base = iter * 4; + if(!sliceEqualsIgnoreCase(raw, slices[base], slices[base + 1], name)) { + continue; + } + if(sliceHasToken(raw, slices[base + 2], slices[base + 3], token)) { + return true; + } + } + return false; + } + + /** The slice form: no String is built for the field or for its tokens. */ + private boolean sliceHasToken(byte[] data, int start, int length, String token) { + int end = start + length; + int at = start; + while(at < end) { + while(at < end && (data[at] == ' ' || data[at] == '\t' || data[at] == ',')) { + at++; + } + int tokenStart = at; + while(at < end && data[at] != ',') { + at++; + } + int tokenEnd = at; + while(tokenEnd > tokenStart + && (data[tokenEnd - 1] == ' ' || data[tokenEnd - 1] == '\t')) { + tokenEnd--; + } + if(tokenEnd - tokenStart == token.length() + && sliceEqualsIgnoreCase(data, tokenStart, tokenEnd - tokenStart, token)) { + return true; + } + } + return false; + } + + /** The String form, for a Request built from a map rather than a socket. */ + private boolean listHasToken(String value, String token) { + int at = 0; + while(at <= value.length()) { + int comma = value.indexOf(',', at); + int end = comma < 0 ? value.length() : comma; + int start = at; + while(start < end && (value.charAt(start) == ' ' || value.charAt(start) == '\t')) { + start++; + } + int trimmed = end; + while(trimmed > start + && (value.charAt(trimmed - 1) == ' ' || value.charAt(trimmed - 1) == '\t')) { + trimmed--; + } + // regionMatches(true, ...) compares character by character and is + // locale independent, unlike folding both sides with toLowerCase(). + if(trimmed - start == token.length() + && value.regionMatches(true, start, token, 0, token.length())) { + return true; + } + if(comma < 0) { + return false; + } + at = comma + 1; + } + return false; + } + + /** How many headers arrived, so a duplicate can be detected. */ + int countHeader(String name) { + int found = 0; + for(int iter = 0 ; iter < headerCount ; iter++) { + int base = iter * 4; + if(sliceEqualsIgnoreCase(raw, slices[base], slices[base + 1], name)) { + found++; + } + } + return found; + } + + public String getBody() { + return body; + } + } + + /** What a handler returns. */ + public static final class Response { + // Not final because Request.respond hands back ONE Response per connection, + // re-pointed per request. The same trade the Request beside it already + // makes: valid for the duration of Handler.handle and not beyond it, which + // is the whole window in which a handler can see it. A handler that would + // rather own its Response still writes new Response(...) and pays for it. + int status; + String contentType; + byte[] body; + /** When >= 0 the body is this descriptor, and the server owns closing it. */ + int fileFd; + long fileOffset; + long fileLength; + Map extraHeaders; + /** Serialised into the connection buffer at write time; see jsonValue. */ + Object deferredJson; + boolean hasDeferredJson; + + /** + * Re-points this Response. Every field is assigned with no "unchanged" + * case: a field left behind describes the PREVIOUS response on this + * connection, and deferredJson is the one that would hurt -- it makes the + * writer serialise an object the handler never returned. + */ + void reset(int status, String contentType, byte[] body, int fileFd, + long fileOffset, long fileLength, Map extraHeaders) { + this.status = status; + this.contentType = contentType; + this.body = body == null ? EMPTY_BODY : body; + this.fileFd = fileFd; + this.fileOffset = fileOffset; + this.fileLength = fileLength; + this.extraHeaders = extraHeaders; + this.deferredJson = null; + this.hasDeferredJson = false; + } + + public Response(int status, String contentType, byte[] body) { + this(status, contentType, body == null ? new byte[0] : body, -1, 0, 0, null); + } + + /** + * A body AND application headers, which nothing public could express. + * + * The constructor above always passed null for them, empty() takes headers + * but discards the body, and file() wants a descriptor -- so a handler + * returning JSON with a Set-Cookie, a CORS header or a cache directive had + * no supported way to say so, even though both protocol writers send extra + * headers. Server-owned names are still refused at write time. + */ + public Response(int status, String contentType, byte[] body, Map extraHeaders) { + this(status, contentType, body == null ? new byte[0] : body, -1, 0, 0, + extraHeaders); + } + + Response(int status, String contentType, byte[] body, + int fileFd, long fileOffset, long fileLength, Map extraHeaders) { + this.status = status; + this.contentType = contentType; + this.body = body; + this.fileFd = fileFd; + this.fileOffset = fileOffset; + this.fileLength = fileLength; + this.extraHeaders = extraHeaders; + } + + /** + * A response whose body is a file. The server sends it with sendfile where + * the platform has it, so the bytes never enter user space, and CLOSES the + * descriptor when it is done -- a handler that returned one must not. + */ + public static Response file(int status, String contentType, int fd, + long offset, long length, Map extraHeaders) { + return new Response(status, contentType, null, fd, offset, length, extraHeaders); + } + + public static Response text(int status, String body) { + return new Response(status, "text/plain; charset=utf-8", bytes(body)); + } + + public static Response json(int status, String body) { + return new Response(status, "application/json; charset=utf-8", bytes(body)); + } + + /** + * A JSON response serialised straight into the connection's write buffer. + * + * Prefer this to json(status, Json.write(value)): that builds a + * StringBuilder, grows its char[], copies it into a String and encodes + * that to bytes, for a document about to go to a socket and be discarded. + * Deliberately a DIFFERENT NAME rather than an overload taking Object -- + * an overload would bind a String-typed variable to this method and + * double-encode it, which is exactly the kind of thing that is found in + * production rather than in review. + */ + public static Response jsonValue(int status, Object value) { + Response r = new Response(status, "application/json; charset=utf-8", + EMPTY_BODY, -1, 0, 0, null); + r.deferredJson = value; + r.hasDeferredJson = true; + return r; + } + + /** A response with headers but no body, for 304 and for HEAD. */ + public static Response empty(int status, String contentType, Map extraHeaders) { + return new Response(status, contentType, new byte[0], -1, 0, 0, extraHeaders); + } + + public int getStatus() { + return status; + } + + private static byte[] bytes(String s) { + try { + return s == null ? new byte[0] : s.getBytes("UTF-8"); + } catch (IOException err) { + return new byte[0]; + } + } + } + + public interface Handler { + Response handle(Request request) throws Exception; + } + + /** + * How long stop() waits, after closing the sockets, for workers to unwind before + * it releases any session they might still have been inside. + */ + private static final int SESSION_RELEASE_GRACE_MILLIS = 2000; + + private static final int MAX_HEADER_BYTES = 64 * 1024; + + /** + * How much response body one HTTP/2 turn may hold before it drains. + * + * Not a limit on any response, which MAX_BODY_BYTES governs: a limit on how + * many of them may sit copied into native buffers at once while this loop + * keeps answering the next ready stream. + */ + /** + * What a request body buffer starts at, and doubles from as bytes arrive. Not + * the declared Content-Length: see fillTo for why believing that number before + * the body exists is what lets a client allocate memory it never has to send. + */ + private static final int BODY_CHUNK_BYTES = 16 * 1024; + + /** + * Request-body bytes held by uploads IN PROGRESS, across the process. + * + * Growing with the data removed the case where a client allocates 8MB by + * declaring it and sending nothing. It does not bound the case where the + * client really sends nearly all of it on many connections and pauses before + * the last byte: that memory is real, it is held until the rate allowance + * expires, and nothing counted it. The connection ceiling is in the + * thousands, so a modest number of near-complete uploads is the machine. + * + * Scoped to the read, which is what makes it safe to account at all: the + * charge is taken as the buffer grows and given back in a finally on every + * path out of fillTo. A reservation that outlived the call would have to be + * threaded through borrowed thread buffers, owned copies and every failure + * path, and ONE leaked reservation wedges the server for good -- a worse + * failure than the one it fixes. What this bounds is uploads in flight, + * which is the shape of the attack. + */ + private static final java.util.concurrent.atomic.AtomicLong http1UploadBytes = + new java.util.concurrent.atomic.AtomicLong(); + + private static final long MAX_HTTP1_UPLOAD_BYTES = + envInt("CN1_HTTP_MAX_UPLOAD_MB", 64) * 1024L * 1024L; + + private static final long MAX_QUEUED_H2_BODY_BYTES = 4L * 1024 * 1024; + + /** + * File-backed HTTP/2 responses that may be outstanding across the process. + * A separate limit from the byte one because it is a separate resource: such + * a response holds a DESCRIPTOR and no heap, so the byte figure never sees + * it, and a peer that keeps its flow-control window shut holds one per + * stream for as long as it likes. Descriptors run out process-wide, and when + * they do the server stops accepting sockets and opening files entirely -- + * a failure with nothing to do with whoever caused it. + */ + private static final int MAX_OPEN_H2_FILES = envInt("CN1_HTTP_MAX_H2_FILES", 128); + + /** + * Response-body heap that may be outstanding across the PROCESS. The limit + * beside it is per turn and per session, which bounds one connection -- and + * the connection ceiling is in the thousands, so a body per connection is + * still gigabytes. Memory runs out process-wide, so it is counted that way, + * exactly like the descriptors above. + */ + private static final long MAX_OPEN_H2_BODY_BYTES = + envInt("CN1_HTTP_MAX_H2_BODY_MB", 64) * 1024L * 1024L; + private static final int MAX_BODY_BYTES = 8 * 1024 * 1024; + private static final int READY_CAPACITY = 256; + + /** + * Bodies at or below this are sent together with the headers in one write. + * Sized so an ordinary JSON response fits and a page-sized payload does not; + * beyond it the copy costs more than the syscall it saves. + */ + private static final int COMBINED_WRITE_LIMIT = 8192; + + private static final byte[] EMPTY_BODY = new byte[0]; + /** One instance, so the pooled JSON path does not intern a literal per call. */ + static final String JSON_CONTENT_TYPE = "application/json; charset=utf-8"; + /** What a Response with no content type is sent as, on either protocol. */ + static final String DEFAULT_CONTENT_TYPE = "application/octet-stream"; + + /** "Sat, 29 Aug 2026 07:11:02 GMT" -- RFC 9110 fixes the width. */ + private static final int HTTP_DATE_LENGTH = 29; + + /** + * How long a worker waits, still holding the connection, for the NEXT request + * before handing it back to the poller. + * + * This is the difference between a poller that is re-armed per request and one + * that is not. Handing the descriptor back costs an epoll_ctl pair, two + * blocking-mode flips and a cross-thread handoff -- measured against Go, whose + * netpoller registers a connection once: 2 epoll_ctl, 4 fcntl and 3.9 futex + * per request against its 0.00, 0.00 and 0.05, with the futex traffic alone + * 80% of our syscall time. + * + * A client that is going to send another request usually sends it within + * microseconds, so a few milliseconds captures nearly all of them. Set to 0 to + * hand back immediately, which is the behaviour this replaces. + * + * The timeout alone does NOT bound how long a worker keeps a connection: a + * client that keeps sending is readable every time, so the worker goes round + * again and holds it indefinitely. Under continuous load that made the pool + * the limit on concurrent clients -- exactly what the reactor exists to + * prevent -- and it was invisible in a throughput number, because the + * connections that DID hold a worker were served at full speed while the rest + * starved. A fresh connection got no response in five seconds while the + * benchmark reported 234k requests a second. What bounds it is + * {@link #pendingWork} below, plus the burst cap. + */ + /** + * Which thread takes a ready descriptor from the poller. + * + * 0 A dedicated reactor thread calls the poller and DISPATCHES: it + * deregisters the descriptor, allocates a task, queues it and wakes a + * worker. That wake is a futex and a context switch, and the descriptor + * has to be registered again afterwards, so an ordinary request costs + * two epoll_ctl and a cross-thread handoff on top of its own read and + * write. + * 1 The WORKERS call the poller themselves. A worker with nothing to do + * waits on the same set and serves the first descriptor it is given, on + * the thread that polled -- no queue, no wake, no task object. The + * descriptor is armed {@link Reactor#ONESHOT} so the kernel hands it to + * exactly one waiter, and re-arming afterwards is a single epoll_ctl. + * + * Mode 1 is what Go's scheduler does. `netpoll()` is called from + * `findRunnable()` on whatever thread has run out of work, and the result is + * `gp := list.pop(); injectglist(&list); return gp` -- it runs the first + * ready goroutine ON THE POLLING THREAD and only queues the remainder. A + * syscall census of the two servers under the same load put us at 6.9x Go's + * futex rate and 41x its epoll_ctl rate while the read and write counts + * matched to within 5%, which says the gap is coordination rather than work. + */ + /** + * 2 ONE worker polls at a time. It serves the first ready descriptor on + * its own thread and queues the remainder for the others, so a lone + * event -- the common case -- costs no handoff at all, while a burst + * pays one wake per SURPLUS descriptor rather than one per request. + * + * Mode 2 is what Go actually does, and mode 1 is what it looks like from a + * distance. The difference is a guard in `findRunnable` that mode 1 has no + * equivalent of: "we can safely skip it if there are no waiters or A THREAD + * IS BLOCKED IN NETPOLL ALREADY". Go never has two threads in the poller. + * Mode 1 puts every worker in `epoll_wait` on one set, so a single arriving + * event wakes all of them; ONESHOT still guarantees only one RECEIVES the + * descriptor, but the other wakeups happen anyway and cost more the more + * workers there are. Measured, that is exactly what mode 1 does: +40% on two + * workers, +24% on four, and -25% on eight. + */ + /** + * 3 A VIRTUAL THREAD per connection. Host threads poll and resume; a + * connection's virtual thread runs until it finishes or asks for bytes + * that have not arrived, and parks inside the ordinary blocking read. + * There is no handoff at all, and no thread per connection either. + * + * The numbers that motivate mode 3 rather than more tuning of 0 to 2: moving + * a request between OS threads measured 21181ns on the machine this was built + * on, switching a virtual thread measured 2.6ns, and the paired experiment + * over modes 0 to 2 showed the handoff is worth about a third of throughput + * at four workers while REMOVING it costs about a third at sixteen -- because + * a pool large enough to hide the handoff is a pool large enough to lose to + * the OS scheduler. A virtual thread is how a context per connection stops + * implying an OS thread per connection, which is the assumption that made + * those two facts irreconcilable. + */ + /** + * 0 the reactor thread dispatches to a pool; 3 a virtual thread per connection. + * + * Modes 1 and 2 were two ways of letting the WORKERS poll, and the paired + * experiment killed both: removing the dispatch is worth about a third of + * throughput at four workers and costs about a third at sixteen, because a + * pool big enough to hide the handoff is a pool big enough to lose to the OS + * scheduler. Their numbers are in the benchmarks README; the code is gone + * rather than left to rot, since a mode nobody selects is a mode nobody + * tests. + */ + /* + * The DEFAULT is virtual threads wherever the build has them, and the pool + * only where it does not. + * + * Virtual threads are not a tuning option here, they are the mode that + * matches Go on the TAIL: measured p99 1.58 ms against the pool's 59.70 ms on + * the same host, and a corrected syscall census puts their futex traffic at + * 0.000 per request against the pool's 0.265. Leaving the pool as the default + * shipped the worse tail to everyone who did not know to set an environment + * variable, and left the better path exercised only by benchmarks. + * + * It is a TRADE, not a free win, and the cost is throughput. Four arms + * interleaved on a quiet host, /plaintext at 64 connections, medians of four + * steady-state reps (spread 0.4-4.9%): + * + * go 243,161 req/s + * pool, 64 workers 239,155 0.972 of go + * virtual threads 207,808 0.854 of go + * + * So the pool is within 3% of Go on throughput and virtual threads are 13% + * behind it. That gap is NOT syscalls -- the census has virtual threads at + * 3.02 per request against the pool's 3.29 -- so it is user-space switch and + * scheduling cost, which is where to look next if this default is to stop + * costing anything. + * + * Conditioned on VirtualThread.supported() rather than assumed: the context + * switch is compiled in only on non-Windows aarch64/x86_64, and elsewhere + * create() can only return 0, which would drop every connection instead of + * falling back. Setting CN1_HTTP_POLL_MODE explicitly still overrides this + * in either direction. + */ + private static final int POLL_MODE = + envInt("CN1_HTTP_POLL_MODE", VirtualThread.supported() ? 3 : 0); + private static final boolean VIRTUAL_THREADS = POLL_MODE == 3; + + /** + * C stack per connection. Java locals and the operand stack are NOT here -- + * they live in the virtual thread's own VM state, mapped lazily -- so this + * buys call depth rather than data. 64KB holds a few hundred nested Java + * frames, well past what an HTTP handler needs, and is mapped lazily too. + */ + private static final int VT_STACK_BYTES = envIntAtLeast("CN1_HTTP_VT_STACK", 64 * 1024, 1); + + /** + * How often a virtual-thread host sweeps its deadlines, however busy it is. + * The same 250ms the idle poll waits, so a quiet host behaves exactly as + * before and a busy one stops being exempt. + */ + private static final long SWEEP_INTERVAL_MILLIS = 250; + + private static final int KEEPALIVE_LINGER_MILLIS = + envInt("CN1_HTTP_KEEPALIVE_LINGER_MS", 5); + + /** + * How long a worker waits for a request, and for the client to take the + * response. A connection that opens and says nothing would otherwise hold a + * worker forever, and the pool is bounded on purpose -- open as many silent + * connections as there are workers and the server stops answering anyone. + */ + private static final int SOCKET_TIMEOUT_MILLIS = envInt("CN1_HTTP_TIMEOUT_MS", 15000); + + /** + * The slowest upload this server will wait for, in bytes per second. + * + * 8 KB/s is well under any real link and still bounds a body: 8 MiB has about + * seventeen minutes to arrive, and a client sending a byte at a time does not + * get them. Set CN1_HTTP_MIN_BODY_RATE to change it. + */ + private static final int MIN_BODY_BYTES_PER_SECOND = + envIntAtLeast("CN1_HTTP_MIN_BODY_RATE", 8192, 1); + + /** + * Ceiling on open connections. Past it a connection is accepted and closed + * immediately rather than left in the backlog: refusing is a fast, legible + * answer, while a full backlog looks to a client like a server that hangs. Set + * to 0 for no ceiling. + */ + private static final int MAX_CONNECTIONS = envInt("CN1_HTTP_MAX_CONNECTIONS", 4096); + + /** + * A tunable that must be at least `minimum`, or the default is used instead. + * + * Some of these settings are DIVISORS or array sizes, and a zero reaches very + * different places on the two runtimes: Java SE throws ArithmeticException, + * which the reader catches as an ordinary read failure and drops the + * connection with no response, while ParparVM answers 0 for an integer + * division by zero -- so the packaged binary silently loses the rate part of + * its own deadline instead. Neither is what anyone typed 0 hoping for, and a + * setting that behaves differently in the dev loop than in production is the + * exact divergence this backend keeps being reviewed for. + * + * Package-visible so the runtime self-test can check the clamp itself on both + * arms; the values it guards are read once at class initialisation, which no + * test can reach. + */ + static int atLeast(String name, int value, int minimum) { + if(value >= minimum) { + return value; + } + System.err.println(name + "=" + value + " is below the minimum of " + minimum + + "; using the default instead"); + return -1; + } + + private static int envIntAtLeast(String name, int fallback, int minimum) { + int value = envInt(name, fallback); + return atLeast(name, value, minimum) < 0 ? fallback : value; + } + + private static int envInt(String name, int fallback) { + String v = System.getenv(name); + if(v == null || v.length() == 0) { + return fallback; + } + try { + return Integer.parseInt(v.trim()); + } catch (NumberFormatException err) { + return fallback; + } + } + + /** + * Set CN1_HTTP_TRACE=1 to print what the reactor sees. A reactor that is not + * reporting readiness looks exactly like a handler that is not responding, and + * the only way to tell them apart from outside is to ask which one is silent. + */ + private static final boolean TRACE = "1".equals(System.getenv("CN1_HTTP_TRACE")); + + private static void trace(String message) { + if(TRACE) { + System.err.println("[http] " + message); + } + } + + private final ServerSocket listener; + private final Reactor reactor; + private final ExecutorService workers; + private final Handler handler; + private final Tls tls; + /** + * fd to SSL session. Only written when a connection is established or closed, + * never per request. A TLS connection genuinely costs an object; the plain + * server allocates nothing per idle connection and this map is why that + * property does not carry over to TLS. + */ + private final Map sessions = java.util.Collections.synchronizedMap(new java.util.HashMap()); + /** fd to HTTP/2 session, for connections where ALPN settled on h2. */ + private final Map http2Sessions = java.util.Collections.synchronizedMap(new java.util.HashMap()); + /** + * Every accepted descriptor that has not been dropped yet. + * + * The TLS and HTTP/2 maps only hold the connections that have one of those, so + * a plaintext connection appeared in neither and stop() had nothing to close it + * with. It would stay open past the drain deadline while the server reported + * itself fully stopped. + */ + private final Map liveConnections = java.util.Collections.synchronizedMap(new java.util.HashMap()); + + /** + * When each PARKED pooled connection stops being worth keeping. + * + * The virtual-thread path has this on its hosts, keyed by descriptor and swept + * by the poller. The pooled reactor had nothing: it registered the descriptor + * and left, and SO_RCVTIMEO cannot expire a socket while no thread is inside + * recv, so an accepted connection that said nothing -- or a keep-alive one + * re-armed and then abandoned -- stayed in liveConnections for ever. Enough of + * them reach MAX_CONNECTIONS and every later client is refused, which is the + * cheapest denial of service there is. Every Java SE run and every TLS server + * takes this path. + * + * Only while PARKED: handOff removes the entry, because a connection a worker + * is serving is bounded by the request deadlines instead. + */ + private final Map pooledDeadlines = + java.util.Collections.synchronizedMap(new java.util.HashMap()); + private volatile boolean running = true; + private Thread loop; + /** Released only when stop() has finished draining. See awaitTermination. */ + private final Object stopped = new Object(); + private boolean fullyStopped; + private final java.util.concurrent.atomic.AtomicInteger openConnections = + new java.util.concurrent.atomic.AtomicInteger(); + /** + * Requests actually being served, as opposed to connections being held. + * + * activeRequests counts a worker's whole stay on a connection, which the pool + * sizing below genuinely wants -- but in virtual-thread mode a worker owns a + * keep-alive connection for its lifetime and parks between requests, so that + * number stays positive while the client sits idle. Reported as saturation it is + * wrong, and stop() waiting on it meant one idle keep-alive client held shutdown + * for the entire drain window. + */ + /** + * HTTP/2 turns inside nghttp2, which stop() has to wait for as well. + * + * Separate from inFlightRequests rather than folded into it: that one is what + * getMetrics reports as active requests, and a connection pumping control + * frames or flushing after its last stream is not a request in flight -- but + * it IS a reason not to free the session under it. + */ + private final java.util.concurrent.atomic.AtomicInteger http2Turns = + new java.util.concurrent.atomic.AtomicInteger(); + + private final java.util.concurrent.atomic.AtomicInteger inFlightRequests = + new java.util.concurrent.atomic.AtomicInteger(); + + private final java.util.concurrent.atomic.AtomicInteger activeRequests = + new java.util.concurrent.atomic.AtomicInteger(); + /** + * Requests answered, striped one slot per host thread. + * + * A profile put AtomicLong.incrementAndGet among the hottest symbols in this + * server: requestsServed was one CONTENDED atomic per request, and with the + * host threads pinned to two cores every increment moved a cache line between + * them. Each slot here has a single writer -- the host thread that owns the + * connection -- so the increment is a plain add, and the health endpoint sums + * the stripes. Slots are 8 longs apart so two hosts never share a cache line, + * which is the whole point of striping and easy to leave out by accident. + * + * Kept alongside requestsServed rather than replacing it: the reactor mode has + * no hosts to stripe by, and still uses the atomic. + */ + private static final int SERVED_STRIPE_STRIDE = 8; + private long[] servedStripes = new long[0]; + + private long servedTotal() { + long total = requestsServed.get(); + long[] st = servedStripes; + for(int i = 0 ; i < st.length ; i += SERVED_STRIPE_STRIDE) { + total += st[i]; + } + return total; + } + + private final java.util.concurrent.atomic.AtomicLong requestsServed = + new java.util.concurrent.atomic.AtomicLong(); + private final java.util.concurrent.atomic.AtomicLong connectionsAccepted = + new java.util.concurrent.atomic.AtomicLong(); + private final java.util.concurrent.atomic.AtomicLong connectionsRefused = + new java.util.concurrent.atomic.AtomicLong(); + private final long startedAt = System.currentTimeMillis(); + + + /** + * How many requests one worker may serve on one connection before handing it + * back even when nothing else is waiting. + * + * A backstop under the pendingWork check rather than the main mechanism: it + * bounds the damage if that check is ever wrong. In virtual-thread mode the + * cap still applies but its ACTION is to step aside rather than to close -- + * see where it is used. + */ + private static final int KEEPALIVE_BURST_LIMIT = + envInt("CN1_HTTP_KEEPALIVE_BURST", 256); + + /** How many requests may be in flight at once; the pool size. */ + private final int workerCount; + + /** + * Whether THIS server runs on virtual threads, which is not the same question as + * whether the build supports them. + * + * A TLS server does not, however POLL_MODE is set. Tls.readImpl maps + * SSL_ERROR_WANT_READ to a hard error rather than parking, so a TLS descriptor + * has to stay blocking -- and a blocking descriptor on a virtual thread holds + * its host OS thread for the whole read. With one host per core, one idle TLS + * client per core occupies every host and unrelated connections stop being + * served. A thread pool has a worse ceiling and an honest one; virtual threads + * here have a better ceiling that a single slow client removes. + * + * So TLS falls back to the pool until the TLS layer can park. This is decided + * once, here, rather than tested at each use, because half a server in each mode + * is neither. + */ + private final boolean virtualThreads; + + private HttpServer(ServerSocket listener, Reactor reactor, ExecutorService workers, + int workerCount, Handler handler, Tls tls) { + this.listener = listener; + this.reactor = reactor; + this.workers = workers; + this.workerCount = workerCount; + this.handler = handler; + this.tls = tls; + // Derived from the decision the caller actually made, not recomputed from + // the statics behind it. Recomputing was right while "plaintext" was the + // only condition, but a server can now fall back to the pool for a second + // reason -- another server already holds the single virtual-thread slot -- + // and a server that recomputed would have run on this pool while believing + // itself virtual, which changes parking, keep-alive linger, ownership and + // teardown. No pool means virtual threads; that is the whole of it. + this.virtualThreads = workers == null; + } + + /** + * Arming used for connection descriptors. + * + * Plain level-triggered READ, with no ONESHOT and so no re-arm, and affinity + * is what makes that safe. A descriptor lives in exactly ONE host's epoll + * set, and that host is not polling while it is inside advance() running the + * virtual thread, so no second thread can ever be handed a descriptor whose + * virtual thread is already running. ONESHOT was guarding against a hazard + * that only exists when several threads share a poller. + * + * What it cost to keep it was an epoll_ctl on every park, which is the exact + * syscall Go does not pay: it registers each descriptor once and never + * touches epoll again for the life of the connection. A profile of the + * plaintext benchmark put epoll_ctl at 4.65% of in-binary self time, so the + * re-arm is now gone and a descriptor stays armed for the whole connection. + * + * ONESHOT was doing one more thing than the hazard above, and dropping it + * without replacing that is a use-after-free. The kernel DISARMS on delivery, + * so a descriptor whose virtual thread returned RUNNABLE -- queued in the + * ring, neither running nor parked -- could not be reported again while it + * sat there. Left armed it can be, and advance() would resume a handle the + * ring is also about to resume. That invariant is now explicit: the RUNNABLE + * path disarms with a remove() and VtHost.armedByFd remembers it, which costs + * a syscall on the yield path instead of on every request. + */ + private static final int CONN_EVENTS = Reactor.READ; + + /** + * Ready descriptors handed to the pool and not yet picked up. + * + * The keep-alive linger is bounded by this rather than by its timeout: a + * client that keeps sending is readable every time, so a worker would hold + * one connection for ever and the pool would become the limit on concurrent + * clients. A fresh connection got no response in five seconds while the + * benchmark reported 234k requests a second. + */ + private final java.util.concurrent.atomic.AtomicInteger pendingWork = + new java.util.concurrent.atomic.AtomicInteger(0); + + /** Set only when a poller-per-worker mode is on; the threads that poll. */ + private Thread[] pollers; + + private final java.util.concurrent.atomic.AtomicInteger vtAccepts = + new java.util.concurrent.atomic.AtomicInteger(0); + private final java.util.concurrent.atomic.AtomicInteger vtDispatched = + new java.util.concurrent.atomic.AtomicInteger(0); + private final java.util.concurrent.atomic.AtomicInteger vtCreateFailures = + new java.util.concurrent.atomic.AtomicInteger(0); + + public static HttpServer start(String host, int port, int backlog, int workerCount, Handler handler) + throws IOException { + return start(host, port, backlog, workerCount, handler, null); + } + + /** + * - `tls`: terminate TLS here, or null to serve plaintext (correct behind a + * load balancer that already terminated it) + */ + public static HttpServer start(String host, int port, int backlog, int workerCount, + Handler handler, Tls tls) throws IOException { + ServerSocket listener = ServerSocket.bind(host, port, backlog); + Reactor reactor; + try { + reactor = Reactor.create(); + } catch (IOException err) { + listener.close(); + throw err; + } + ServerSocket.setBlocking(listener.getFd(), false); + reactor.add(listener.getFd(), CONN_EVENTS); + + // No worker pool in virtual-thread mode. workers.execute() is reached only + // from handOff(), which is reached only from pump(), which runs only in + // the branch below this one -- so in this mode every pooled thread is + // created, parked, and never given anything to do. + // + // They are not free. Each is a Java thread of control: the collector + // conservatively scans its native stack and its 258KB object stack on + // every cycle, and stop-the-world sets threadBlockedByGC on each and + // spins `while(t->threadActive)` waiting for it. Measured on two pinned + // cores with the host count held equal by the clamp above -- so the pool + // size was the only variable -- WORKERS=64 segfaulted 2 runs in 6 and + // WORKERS=4 survived 6 of 6. Throughput was unaffected when it did not + // crash (265k either way), so this buys robustness rather than speed. + boolean useVirtualThreads = VIRTUAL_THREADS && tls == null; + // The native virtual thread carries the accepted descriptor and nothing + // else, so the Java side finds its server through one process-global. A + // second virtual-thread server would replace it, and every connection the + // FIRST listener had accepted would then be served by the second one's + // handler and ownership maps -- an administrative port answering public + // requests, and neither able to shut down what it owns. Only one server + // can hold that slot; the next takes the pool, which is per instance and + // has no such ambiguity. Claimed before the server is built so two + // starting at once cannot both win it. + if(useVirtualThreads && !VT_SLOT_TAKEN.compareAndSet(false, true)) { + System.out.println("another virtual-thread server is already running in " + + "this process, so this one runs on a thread pool: an accepted " + + "descriptor is all a virtual thread carries, and it cannot say " + + "which server to hand it to."); + useVirtualThreads = false; + } + if(VIRTUAL_THREADS && tls != null) { + System.out.println("TLS is configured, so this server runs on a thread " + + "pool rather than virtual threads: the TLS layer cannot park a " + + "read yet, and a blocking read on a virtual thread holds its " + + "host for the duration."); + } + final HttpServer server = new HttpServer(listener, reactor, + useVirtualThreads ? null : Executors.newFixedThreadPool(workerCount), + workerCount, handler, tls); + if(useVirtualThreads) { + ACTIVE_SERVER = server; + // A poller PER HOST, because affinity is enforced by the poller: a + // descriptor registered in one host's set can only ever be reported + // to that host, so its virtual thread cannot run anywhere else. The + // listener lives in host 0's set, so exactly one host accepts and + // hands each new connection to its permanent owner. + // HOSTS TRACK CORES, not the caller's expected concurrency. + // + // In this mode workerCount stops meaning "how many requests may be in + // flight" -- the virtual threads supply that, one per connection -- + // and a host thread is only useful while there is a core free to run + // it on. Past that they contend for the cores the server needs: + // measured on two pinned cores, 16 hosts served 117 requests where 2 + // served 257297. Unpinned, with cores to spare, 2 through 32 all + // behave, so the ceiling has to come from the machine at runtime. + // + // Clamped rather than obeyed, because a caller asking for 64 workers + // is asking for concurrency, and in this mode that request is + // answered by the virtual threads instead. + int hostCount = workerCount; + int cores = ServerSocket.availableProcessors(); + if(hostCount > cores) { + hostCount = cores; + } + if(hostCount < 1) { + hostCount = 1; + } + server.vtHosts = new VtHost[hostCount]; + // One cache line per host, so the stripes never share one. + server.servedStripes = new long[hostCount * SERVED_STRIPE_STRIDE]; + for(int iter = 0 ; iter < hostCount ; iter++) { + server.vtHosts[iter] = new VtHost(iter == 0 ? reactor : Reactor.create()); + } + server.pollers = new Thread[hostCount]; + for(int iter = 0 ; iter < hostCount ; iter++) { + final int index = iter; + server.pollers[iter] = new Thread(new Runnable() { + public void run() { + server.runVirtualThreadHost(index); + } + }); + server.pollers[iter].start(); + } + } else { + server.loop = new Thread(new Runnable() { + public void run() { + server.pump(); + } + }); + server.loop.start(); + } + return server; + } + + public int getPort() { + return listener.getPort(); + } + + /** Connections currently open. */ + public int getOpenConnections() { + return openConnections.get(); + } + + /** Requests being handled right now. This is what saturation looks like. */ + public int getActiveRequests() { + return inFlightRequests.get(); + } + + /** + * A snapshot for a health or metrics endpoint. "draining" is what a load + * balancer needs to see to take this instance out of rotation before it stops + * answering. + */ + public Map getMetrics() { + Map out = new LinkedHashMap(); + out.put("status", running ? "ok" : "draining"); + out.put("uptimeSeconds", new Long((System.currentTimeMillis() - startedAt) / 1000L)); + out.put("openConnections", new Integer(openConnections.get())); + out.put("activeRequests", new Integer(inFlightRequests.get())); + out.put("requestsServed", new Long(servedTotal())); + out.put("connectionsAccepted", new Long(connectionsAccepted.get())); + out.put("connectionsRefused", new Long(connectionsRefused.get())); + out.put("tls", tls == null ? "off" : "on"); + out.put("http2Connections", new Integer(http2Sessions.size())); + // A descriptor handed to a Response and not yet closed. Reported + // because nothing else can see one that escapes: the process limit is + // enormous, so a leak surfaces hours later as a server that cannot + // accept sockets, with nothing pointing at the cause. + out.put("openStaticFiles", new Integer(StaticFiles.openFileCount())); + return out; + } + + /** + * Blocks until the server has fully stopped, draining included. + * + * A caller's main() must do this or something equivalent: the reactor and the + * workers run on threads ParparVM creates DETACHED, so when main returns the + * process exits and takes them with it -- silently, with status 0, which from + * outside looks exactly like a server that refuses connections. + * + * Waiting on the reactor THREAD is not enough, and that was a real bug: stop() + * clears the running flag, the loop returns on its next timeout, main wakes up + * and the process ends while a worker is still writing a response. This waits + * on the drain finishing instead. + */ + public void awaitTermination() { + synchronized(stopped) { + while(!fullyStopped) { + try { + stopped.wait(); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + return; + } + } + } + } + + /** + * Stops accepting, lets in-flight requests finish, then closes what is left. + * + * The order matters. Closing the listener first means no new work arrives while + * the pool drains; draining before closing connections means a request already + * being served gets to produce its response instead of having the socket pulled + * out from under it, which is what a client sees as a truncated reply. + */ + public void stop(int drainMillis) { + running = false; + reactor.remove(listener.getFd()); + listener.close(); + if(workers != null) { // null in virtual-thread mode; see start() + workers.shutdown(); + } + long deadline = System.currentTimeMillis() + drainMillis; + // Waits on requests IN FLIGHT, not on open connections: an idle keep-alive + // connection has nothing to finish and would otherwise hold the shutdown + // open for the whole window for no reason. + // + // http2Turns as well, which this loop was missing when that counter was + // added: a turn holds the session and is not a request in flight, so a + // connection pumping frames was not waited for here at all. + // + // NOT covered, deliberately: a response already handed to nghttp2 whose + // DATA frames are still waiting on the peer's flow-control window. No + // worker is inside that connection, so nothing here can see it -- and the + // way to see it, asking the session whether it still wants to write, means + // calling into nghttp2 from THIS thread while a worker may be inside the + // same session, which is the race the descriptor-first teardown below + // exists to avoid. Answering it safely needs the worker to record the + // answer at the end of its own turn; until then such a response can still + // be cut short by a stop(), and that is a smaller fault than a native data + // race during shutdown. + while(System.currentTimeMillis() < deadline + && (inFlightRequests.get() > 0 || http2Turns.get() > 0)) { + try { + Thread.sleep(20); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + break; + } + } + // Whatever is still open at the deadline is an idle keep-alive connection or + // a request that overran; both have to be closed rather than held forever. + // + // The DESCRIPTOR first, and only the descriptor. A worker past the deadline + // may be sitting inside SSL_read or nghttp2 on this very connection, and + // freeing the session under it is a native use-after-free -- a crash during + // shutdown, which is exactly when the remaining work is least recoverable. + // Closing the socket instead unblocks that worker: its next read fails, and + // it takes its own connection down through drop(), which frees the session on + // the thread that was using it. + java.util.Iterator live = new java.util.ArrayList(liveConnections.keySet()).iterator(); + while(live.hasNext()) { + ServerSocket.closeFd(((Integer)live.next()).intValue()); + } + // Then give those workers a moment to notice and unwind. Freeing a session + // while one is still inside it is the thing being avoided, so the sweep below + // waits for the count to reach zero rather than assuming it has. + long freeBy = System.currentTimeMillis() + SESSION_RELEASE_GRACE_MILLIS; + while(System.currentTimeMillis() < freeBy + && (inFlightRequests.get() > 0 || http2Turns.get() > 0)) { + try { + Thread.sleep(20); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + break; + } + } + // The wait above ends on the count reaching zero OR on the grace period + // expiring, and only the first of those means nothing is running. A + // handler still inside a request holds the very TLS and HTTP/2 sessions + // the sweeps below free, and would then write its response through freed + // native memory -- so when the window expired with work outstanding, the + // sessions are left alone. That leaks one per live connection, which a + // process about to exit does not care about and a use-after-free is not + // a trade for. + if(inFlightRequests.get() > 0 || http2Turns.get() > 0) { + releaseVirtualThreadSlot(); + synchronized(stopped) { + fullyStopped = true; + stopped.notifyAll(); + } + return; + } + // Anything still registered had no worker to take it down -- an idle + // connection in reactor mode, where nothing runs for it once its descriptor + // is gone. With no request in flight there is no one left to race, so these + // are safe to release here, and leaving them would leak a native session per + // connection for the life of the process. + // The parked VIRTUAL THREADS first, because drop() cannot reclaim them. + // A connection parked between requests holds a handle in its host's table + // and no request in flight, so the wait above finds nothing to wait for and + // comes straight here. drop() then closes the descriptor and returns, + // leaving the handle -- and its native stack and VM thread registration -- + // allocated. sweepDeadlines is what normally frees those, and it runs from + // the poll loop, which `running = false` has already ended. The process + // keeps every one of them, which matters precisely because + // releaseVirtualThreadSlot() exists so a server CAN be started again here. + freeParkedVirtualThreads(); + java.util.Iterator stranded = new java.util.ArrayList(liveConnections.keySet()).iterator(); + while(stranded.hasNext()) { + drop(((Integer)stranded.next()).intValue()); + } + // Belt and braces: a session recorded for a descriptor that was already + // dropped would otherwise never be freed. + synchronized(sessions) { + java.util.Iterator it = new java.util.ArrayList(sessions.keySet()).iterator(); + while(it.hasNext()) { + Object key = it.next(); + Object session = sessions.remove(key); + if(session != null) { + Tls.closeSession(((Long)session).longValue()); + } + } + } + synchronized(http2Sessions) { + java.util.Iterator it = new java.util.ArrayList(http2Sessions.keySet()).iterator(); + while(it.hasNext()) { + Object h2 = http2Sessions.remove(it.next()); + if(h2 != null) { + ((Http2)h2).close(); + } + } + } + if(tls != null) { + tls.close(); + } + releaseVirtualThreadSlot(); + synchronized(stopped) { + fullyStopped = true; + stopped.notifyAll(); + } + } + + /** + * Hands the single virtual-thread slot back, so a server started later in this + * process can have it. Only the holder releases it: a second server that fell + * back to the pool must not free the running one's claim when it stops. + */ + /** + * Frees every virtual thread still parked on a connection, at shutdown. + * + * Only safe because nothing is running by the time it is called: the poll loop + * has stopped, so no host can resume one of these handles, and a handle that is + * freed while its thread could still be resumed is a use-after-free -- the same + * hazard the RUNNABLE path guards with poller.remove(). + */ + private void freeParkedVirtualThreads() { + VtHost[] hosts = vtHosts; + if(hosts == null) { + return; + } + for(int h = 0 ; h < hosts.length ; h++) { + VtHost host = hosts[h]; + if(host == null) { + continue; + } + for(int fd = 0 ; fd < host.vtByFd.length ; fd++) { + long handle = host.handleFor(fd); + if(handle != 0) { + host.setHandle(fd, 0); + host.setDeadline(fd, 0); + VirtualThread.free(handle); + } + } + } + } + + private void releaseVirtualThreadSlot() { + if(virtualThreads) { + ACTIVE_SERVER = null; + VT_SLOT_TAKEN.set(false); + } + } + + /** Stops with a default drain window. */ + public void stop() { + stop(10000); + } + + private void pump() { + trace("reactor thread started"); + int[] ready = new int[READY_CAPACITY]; + int listenFd = listener.getFd(); + while(running) { + int n; + try { + // A timeout rather than an infinite wait, so stop() is noticed even + // when no connection ever arrives. + n = reactor.await(ready, 250); + } catch (IOException err) { + if(running) { + System.err.println("reactor failed: " + err); + } + return; + } + if(n > 0) { + trace("ready=" + n); + } + for(int iter = 0 ; iter < n ; iter++) { + int fd = ready[iter]; + if(fd == listenFd) { + acceptAll(); + } else { + handOff(fd); + } + } + sweepIdlePooledConnections(); + } + } + + /** + * Closes parked pooled connections whose idle deadline has passed. + * + * On the reactor thread, which is the only one that parks them, and after the + * ready set has been dispatched so a descriptor that just became readable is + * never swept on the same turn. await() returns at least every 250ms, so this + * runs often enough without a timer of its own. + */ + private void sweepIdlePooledConnections() { + if(virtualThreads || pooledDeadlines.isEmpty()) { + return; + } + long now = System.currentTimeMillis(); + java.util.Iterator it = + new java.util.ArrayList(pooledDeadlines.entrySet()).iterator(); + while(it.hasNext()) { + java.util.Map.Entry entry = (java.util.Map.Entry)it.next(); + if(((Long)entry.getValue()).longValue() > now) { + continue; + } + int fd = ((Integer)entry.getKey()).intValue(); + pooledDeadlines.remove(entry.getKey()); + trace("idle deadline reached, dropping fd=" + fd); + drop(fd); + } + } + + /** The methods this server routes. Anything else is 501, not a 404. */ + private static boolean isKnownMethod(String method) { + return "GET".equals(method) || "HEAD".equals(method) || "POST".equals(method) + || "PUT".equals(method) || "DELETE".equals(method) || "PATCH".equals(method) + || "OPTIONS".equals(method); + } + + /** + * The server a virtual thread belongs to. + * + * A virtual thread's body is a C function and cannot carry a Java receiver, + * so it arrives at serveVirtual with a descriptor and nothing else. One + * server per process is the shape every backend binary has. + */ + private static volatile HttpServer ACTIVE_SERVER; + + /** Guards ACTIVE_SERVER: exactly one server per process may use virtual threads. */ + private static final java.util.concurrent.atomic.AtomicBoolean VT_SLOT_TAKEN = + new java.util.concurrent.atomic.AtomicBoolean(); + + /** + * What a connection's virtual thread runs. Reached from native code only, + * which is also what keeps it from being dead-code eliminated. + * + * Deliberately just serve(): the existing connection handling, written in + * the blocking style, unchanged. That it now runs on a virtual thread is + * invisible to it, which is the property that makes virtual threads worth + * having rather than a rewrite into callbacks. + */ + static void serveVirtual(int fd) { + HttpServer server = ACTIVE_SERVER; + if(server == null) { + ServerSocket.closeFd(fd); + return; + } + server.serve(fd); + } + + /** + * One host thread's private world: its poller, its connections, its virtual + * threads, its run queue. + * + * NOTHING here is shared, and that is the point. A virtual thread runs on + * the host that accepted its connection and on no other, for its whole life. + * + * WHY AFFINITY IS NOT A TUNING CHOICE. The VM keeps real state per OS + * THREAD: the BiBOP allocator's current page (`bibopCurrent`), the pacing + * claim (`cn1MyPacingClaim`), the mark buffer, `cn1TlsSelf`, and this + * backend's own zero-copy read buffer. A virtual thread that parks on one + * host and resumes on another continues against a different thread's copy of + * all of it. The first version had no affinity and crashed as soon as the + * collector's backpressure started parking virtual threads mid-allocation. + * + * Auditing each of those for migration-safety would be a list that grows + * every time somebody adds a __thread; pinning the virtual thread to its + * host makes every one of them correct by construction, including the ones + * nobody has written yet. + */ + private static final class VtHost { + final Reactor poller; + + /** + * Virtual threads ready to run, as a preallocated ring of raw handles. + * + * NOT a LinkedList of boxed Longs, and this is the single most important + * line in the scheduler. That version allocated twice per yield -- the + * box and the list node -- ON THE HOST THREAD, and a host thread has no + * virtual thread to hand back when the collector's backpressure stops it: + * it just sleeps. Caught with a debugger, the accepting host was sitting + * in cn1PacingPark underneath LinkedList.addLast underneath advance(), + * which is this queue. Nothing was accepted after that, and the failure + * amplifies itself -- the more the collector is behind, the more the + * scheduler allocates trying to cope. + * + * A scheduler's hot path must not allocate, or it becomes a customer of + * the very backpressure it is supposed to be relieving. + */ + long[] ring = new long[256]; + int ringHead = 0; + int ringCount = 0; + + boolean ringEmpty() { + return ringCount == 0; + } + + void ringAdd(long handle) { + if(ringCount == ring.length) { + // Growth allocates, which is why the ring starts big enough that + // it does not happen in steady state: it is bounded by how many + // virtual threads can be mid-yield at once on ONE host. + long[] grown = new long[ring.length * 2]; + for(int iter = 0 ; iter < ringCount ; iter++) { + grown[iter] = ring[(ringHead + iter) % ring.length]; + } + ring = grown; + ringHead = 0; + } + ring[(ringHead + ringCount) % ring.length] = handle; + ringCount++; + } + + /** + * Head first, and NOT the FILO order fasthttp uses. + * + * fasthttp hands a new connection to its most recently released worker -- + * "such a scheme keeps CPU caches hot" -- and the same idea looks like it + * should apply here. It does not, because it is not the same queue. + * fasthttp is choosing among IDLE, INTERCHANGEABLE workers, where nothing + * can starve: whichever it picks, every connection still has a worker. + * This queue holds PENDING RUNNABLE CONTEXTS, and the order decides + * whether a connection runs at all -- work keeps arriving at the tail, so + * taking from the tail lets the head sit. + * + * Measured rather than reasoned, tail-first against head-first, two reps + * at each of 64 and 256 connections: + * + * 64 conns head-first 277152/176814 rps, 1.48/1.39 cores + * tail-first 92603/ 73980 rps, 0.56/0.55 cores + * 256 conns head-first 274451/270569 rps, 1.52/1.51 cores + * tail-first 131821/120500 rps, 0.81/0.74 cores + * + * Tail-first costs two thirds of the throughput and half the machine, + * four readings out of four. Its p99 looks better only because it is + * serving a third of the traffic. Do not re-import this from fasthttp + * without re-reading which queue it applies to. + */ + long ringTake() { + if(ringCount == 0) { + return 0; + } + long handle = ring[ringHead]; + ringHead = (ringHead + 1) % ring.length; + ringCount--; + return handle; + } + /** Descriptor to virtual-thread handle. Only this host touches it. */ + long[] vtByFd = new long[1024]; + + /** + * Whether each descriptor is currently registered with this host's poller. + * + * Without ONESHOT the kernel no longer disarms on delivery, so this is the + * only record of it. Only the owning host reads or writes it, which is the + * same single-writer rule the tables beside it follow. + */ + boolean[] armedByFd = new boolean[1024]; + + /** + * When each parked connection stops being worth waiting for, or 0. + * + * A parked virtual thread is resumed only when its descriptor becomes + * readable, so a client that connects, sends half a request and then goes + * quiet is never resumed and never shed: the connection lives for ever + * and holds a virtual thread and its stacks. That is slowloris, and + * BackendHttpIntegrationTest.shedsIdleConnections tests for it. The + * dispatching path inherits the behaviour from the socket deadline; this + * path has to enforce it, because nothing else will. + */ + long[] deadlineByFd = new long[1024]; + + /** When this host last swept, so a busy one still sheds stale work. */ + long lastSweep; + + VtHost(Reactor poller) { + this.poller = poller; + } + + long handleFor(int fd) { + return fd < vtByFd.length ? vtByFd[fd] : 0; + } + + /** + * Makes room for this descriptor in all three tables. + * + * Every writer calls it, not just setHandle. The tables start at 1024 + * and a process serving the advertised connection ceiling opens numbers + * far past that, so a write that only bounds-CHECKED was a write that + * silently did nothing: an accepted connection above 1024 recorded no + * deadline, and a client that then sent nothing was never swept, because + * the growth happened in setHandle and setHandle only runs once the + * connection has spoken. Silence was the one case it had to cover. + */ + void ensureCapacity(int fd) { + if(fd < vtByFd.length) { + return; + } + int size = vtByFd.length; + while(size <= fd) { + size = size * 2; + } + long[] grown = new long[size]; + System.arraycopy(vtByFd, 0, grown, 0, vtByFd.length); + vtByFd = grown; + long[] grownDeadlines = new long[size]; + System.arraycopy(deadlineByFd, 0, grownDeadlines, 0, deadlineByFd.length); + deadlineByFd = grownDeadlines; + boolean[] grownArmed = new boolean[size]; + System.arraycopy(armedByFd, 0, grownArmed, 0, armedByFd.length); + armedByFd = grownArmed; + } + + void setHandle(int fd, long handle) { + ensureCapacity(fd); + vtByFd[fd] = handle; + if(handle == 0) { + deadlineByFd[fd] = 0; + // The descriptor is being closed, and close() takes it out of the + // epoll set on its own. Clearing here keeps the flag from claiming + // a registration that the next connection to reuse this number + // would not have. + armedByFd[fd] = false; + } + } + + void setDeadline(int fd, long at) { + if(fd < 0) { + return; + } + ensureCapacity(fd); + deadlineByFd[fd] = at; + } + + boolean isArmed(int fd) { + return fd >= 0 && fd < armedByFd.length && armedByFd[fd]; + } + + void setArmed(int fd, boolean armed) { + if(fd >= 0) { + ensureCapacity(fd); + armedByFd[fd] = armed; + } + } + } + + private VtHost[] vtHosts; + + /** + * Which host owns each descriptor, so a connection can be re-armed on the + * poller it actually lives in. + * + * Written only by the accept loop, which runs on host 0 alone, and read only + * by the owning host -- which cannot learn the descriptor exists until the + * registration syscall below has already happened, so the write is published + * before any reader can reach it. + */ + private int[] vtOwnerByFd = new int[1024]; + + /** Round-robin cursor for handing new connections out. Accept thread only. */ + private int nextVtHost; + + private void setVtOwner(int fd, int host) { + if(fd >= vtOwnerByFd.length) { + int size = vtOwnerByFd.length; + while(size <= fd) { + size = size * 2; + } + int[] grown = new int[size]; + System.arraycopy(vtOwnerByFd, 0, grown, 0, vtOwnerByFd.length); + vtOwnerByFd = grown; + } + vtOwnerByFd[fd] = host; + } + + private VtHost ownerOf(int fd) { + int index = fd < vtOwnerByFd.length ? vtOwnerByFd[fd] : 0; + if(index < 0 || index >= vtHosts.length) { + index = 0; + } + return vtHosts[index]; + } + /** Round robin over the hosts, used only by whoever is accepting. */ + private int vtNextHost = 0; + + /** + * One host thread: run whoever is ready, then poll for more. + * + * Everything it touches belongs to it. The run queue is a plain LinkedList + * because no other thread can reach it, and the descriptor table is a plain + * long[] for the same reason -- affinity is what buys that, and it is worth + * more than the lock it saves, because it is also what makes the VM's + * per-thread allocator state correct under a parked virtual thread. + */ + private void runVirtualThreadHost(int index) { + VtHost me = vtHosts[index]; + int[] ready = new int[READY_CAPACITY]; + int listenFd = listener.getFd(); + boolean owner = (index == 0); // only one host accepts + while(running) { + // Runnable virtual threads first: they wait for a turn, not for the + // network, so polling before running them would delay them by the + // whole poll timeout. + boolean ranSome = drainRunnable(me); + int n; + try { + n = me.poller.await(ready, (ranSome || !me.ringEmpty()) ? 0 : 250); + // On ELAPSED TIME, not on an idle poll. Sweeping only when a poll + // came back empty meant a host that always had at least one event + // never swept at all -- and a client can keep that true with a + // trickle of traffic while its other connections sit silent, so + // the deadline that exists to shed them never runs and they + // accumulate to the process ceiling. Busy is exactly when the + // sweep matters. + long now = System.currentTimeMillis(); + if(n == 0 || now - me.lastSweep >= SWEEP_INTERVAL_MILLIS) { + me.lastSweep = now; + sweepDeadlines(me); + } + } catch (IOException err) { + if(running) { + System.err.println("poller failed: " + err); + } + return; + } + for(int iter = 0 ; iter < n ; iter++) { + int fd = ready[iter]; + if(owner && fd == listenFd) { + acceptAll(); + try { + me.poller.modify(listenFd, CONN_EVENTS); + } catch (IOException err) { + if(running) { + System.err.println("could not re-arm the listener: " + err); + } + return; + } + continue; + } + advance(me, fd, me.handleFor(fd)); + } + } + } + + /** + * Run the virtual threads that are ready. True if any were. + */ + private boolean drainRunnable(VtHost me) { + boolean any = false; + int budget = me.ringCount; // one pass, so a busy one cannot starve the poller + while(budget-- > 0 && !me.ringEmpty()) { + long handle = me.ringTake(); + any = true; + advance(me, VirtualThread.descriptorOf(handle), handle); + } + return any; + } + + /** + * Give a connection's virtual thread its turn, creating it on first sight, + * and do whatever its answer asks for. + * + * The three answers are the whole scheduler. FINISHED means the connection is + * over. PARKED_IO means it wants bytes, so the descriptor goes back to this + * host's poller. RUNNABLE means it gave up its turn but is ready now -- it is + * waiting on the collector, not on its socket -- and handing that one to the + * poller would wait for a client that is itself waiting for the response this + * virtual thread still owes it. + */ + private void advance(VtHost me, int fd, long handle) { + if(fd < 0) { + return; + } + if(handle == 0) { + handle = VirtualThread.create(fd, VT_STACK_BYTES); + if(handle == 0) { + // No stack. Serving it on this thread is not an option here: the + // keep-alive wait is indefinite because it expects to park, so + // this host would never poll again. Refuse instead, and say so + // once -- a server quietly dropping to zero is far worse. + if(vtCreateFailures.incrementAndGet() == 1) { + System.err.println("virtual thread creation failed; " + + "refusing connections rather than pinning a host"); + } + drop(fd); + return; + } + me.setHandle(fd, handle); + } + me.setDeadline(fd, 0); // it is running, so it is not idle + int state = VirtualThread.resume(handle); + if(state == VirtualThread.FINISHED) { + me.setHandle(fd, 0); + VirtualThread.free(handle); + return; + } + if(state == VirtualThread.RUNNABLE) { + // Take it out of the poller for as long as it sits in the ring. It is + // neither running nor parked, so a readable descriptor would otherwise + // be reported and resumed here while the ring is about to resume it + // too -- and the second resume of a handle the first one finished and + // freed is a use-after-free. ONESHOT used to make this impossible by + // disarming as it delivered. + me.poller.remove(fd); + me.setArmed(fd, false); + me.ringAdd(handle); + return; + } + // Parked on I/O: start its clock. Nothing else will, and without it a + // half-sent request parks a virtual thread for ever. + me.setDeadline(fd, System.currentTimeMillis() + SOCKET_TIMEOUT_MILLIS); + try { + // Normally already armed and this is no syscall at all, which is the + // point: a keep-alive connection is registered once at accept and + // parks for every later request without touching epoll again. Only a + // descriptor the RUNNABLE path disarmed has to come back, and it comes + // back as an ADD because remove() really deregistered it. + if(!me.isArmed(fd)) { + me.poller.add(fd, CONN_EVENTS); + me.setArmed(fd, true); + } + } catch (IOException err) { + me.setHandle(fd, 0); + VirtualThread.free(handle); + drop(fd); + } + } + + /** + * Close connections whose deadline passed while they were parked. + * + * Swept on the poll timeout rather than per event: a connection making + * progress is resumed by readability long before this runs, so the only + * descriptors it ever finds are the silent ones. + */ + private void sweepDeadlines(VtHost me) { + long now = System.currentTimeMillis(); + for(int fd = 0 ; fd < me.deadlineByFd.length ; fd++) { + long at = me.deadlineByFd[fd]; + if(at == 0 || at > now) { + continue; + } + long handle = me.handleFor(fd); + me.setDeadline(fd, 0); + if(handle != 0) { + me.setHandle(fd, 0); + VirtualThread.free(handle); + } + drop(fd); + } + } + + /** + * Arm a connection descriptor for its next request. + * + * The two modes need different calls and getting it wrong fails quietly in + * both directions, which is why this is one place. Under ONESHOT the kernel + * DISARMS a descriptor as it delivers it but leaves it registered, so coming + * back is EPOLL_CTL_MOD; an EPOLL_CTL_ADD would fail with EEXIST and the + * connection would hang for ever. The dispatching path removed the + * descriptor before handing it over, so there it has to be an ADD. + * + * @param fresh true for a descriptor the poller has never seen -- one just + * accepted -- which is an ADD either way. + */ + /** + * Hand a connection to a host and keep it there. + * + * Every accepted descriptor used to be registered with `reactor`, which IS + * host 0's poller, so every connection lived on host 0 and the other hosts + * polled empty sets for the life of the process. Virtual-thread mode was + * therefore single threaded: measured 0.75 of two pinned cores against the + * pool's 1.61 and Go's 1.49, while costing the LEAST cpu per request of the + * three (5.64 us against 9.78 and 6.87). It was not slower, it was narrower. + * + * Affinity is enforced by the poller -- a descriptor registered in one host's + * set is only ever reported to that host -- so choosing the set at accept + * time is what assigns the owner, and the virtual thread is then created by + * whichever host first sees it. The accept loop never touches another host's + * descriptor table, so that table stays single-writer. + */ + private void armConnection(int fd, boolean fresh) throws IOException { + if(!virtualThreads) { + pooledDeadlines.put(new Integer(fd), + new Long(System.currentTimeMillis() + SOCKET_TIMEOUT_MILLIS)); + reactor.add(fd, CONN_EVENTS); + return; + } + if(fresh) { + int host = nextVtHost; + nextVtHost = host + 1 >= vtHosts.length ? 0 : host + 1; + setVtOwner(fd, host); + vtHosts[host].poller.add(fd, CONN_EVENTS); + vtHosts[host].setArmed(fd, true); + // Its clock starts NOW, not when it first parks. A connection that sends + // nothing never becomes readable, so advance() never runs for it and the + // deadline it would have set never exists -- and SO_RCVTIMEO does not + // close a socket that is only sitting in a poller. Without this, opening + // connections and saying nothing fills MAX_CONNECTIONS and the server + // starts refusing real ones. + vtHosts[host].setDeadline(fd, System.currentTimeMillis() + SOCKET_TIMEOUT_MILLIS); + return; + } + // Re-arm has to name the SAME poller: an epoll set that does not hold + // this descriptor answers a modify with ENOENT, and before connections + // were distributed this was reached through `reactor` and happened to be + // right for every fd. + ownerOf(fd).poller.modify(fd, CONN_EVENTS); + } + + private void acceptAll() { + while(running) { + int fd = listener.accept(); + if(fd < 0) { + return; // drained + } + trace("accepted fd=" + fd); + if(MAX_CONNECTIONS > 0 && openConnections.get() >= MAX_CONNECTIONS) { + // Accept-and-close rather than stop accepting: leaving it in the + // backlog looks to the client like a server that hangs. + trace("at the connection ceiling, refusing fd=" + fd); + connectionsRefused.incrementAndGet(); + ServerSocket.closeFd(fd); + continue; + } + try { + ServerSocket.setBlocking(fd, false); + ServerSocket.setTimeout(fd, SOCKET_TIMEOUT_MILLIS); + // Registered BEFORE the poller can report it. Arming first would let + // another host thread reach drop() for a descriptor this map has not + // heard of yet, and drop declines to close what it does not own. + liveConnections.put(new Integer(fd), Boolean.TRUE); + openConnections.incrementAndGet(); + armConnection(fd, true); + vtAccepts.incrementAndGet(); + connectionsAccepted.incrementAndGet(); + } catch (IOException err) { + // Through drop() so the count it just incremented comes back down. + drop(fd); + } + } + } + + /** + * Takes the descriptor away from the reactor and gives it to a worker. It has + * to leave the poller BEFORE the worker starts reading: this is level + * triggered, so an fd left registered is reported ready again on the next turn + * and two workers end up on one connection. + */ + /** + * How many ready descriptors one wake may carry. + * + * The dispatching path costs a task object and a WAKE per descriptor, and a + * wake is a futex -- a kernel operation whose cost is the same whichever + * language issues it. Measured against Go under the same load this server + * does 6.9x the futex traffic per request while doing the same number of + * reads and writes, so the coordination is the gap rather than the work. + * + * A poller turn that finds N ready descriptors does not need N wakes: one + * worker can be handed the batch and walk it. That divides the dominant cost + * by the batch size, and batches are BIGGEST under exactly the load where + * this server is furthest behind. + * + * 1 restores the old behaviour exactly, which is what makes the comparison + * an A/B rather than a rewrite. + */ + private static final int HANDOFF_BATCH = envInt("CN1_HTTP_HANDOFF_BATCH", 1); + + /** + * Give a whole batch of ready descriptors to ONE worker, in one wake. + * + * Every descriptor still leaves the poller before any of them is read, for + * the same reason the single handoff does it: level-triggered, an fd left + * registered is reported ready again on the next turn and a second worker + * lands on a connection this batch already owns. + */ + private void handOffBatch(final int[] fds, final int count) { + for(int iter = 0 ; iter < count ; iter++) { + reactor.remove(fds[iter]); + pooledDeadlines.remove(new Integer(fds[iter])); + } + pendingWork.addAndGet(count); + try { + workers.execute(new Runnable() { + public void run() { + for(int iter = 0 ; iter < count ; iter++) { + pendingWork.decrementAndGet(); + serve(fds[iter]); + } + } + }); + } catch (RuntimeException err) { + for(int iter = 0 ; iter < count ; iter++) { + pendingWork.decrementAndGet(); + drop(fds[iter]); + } + } + } + + private void handOff(final int fd) { + // It is about to be served, so the idle deadline no longer applies; the + // request deadlines take over from here. + pooledDeadlines.remove(new Integer(fd)); + trace("handOff fd=" + fd); + reactor.remove(fd); + pendingWork.incrementAndGet(); + try { + workers.execute(new Runnable() { + public void run() { + pendingWork.decrementAndGet(); + serve(fd); + } + }); + } catch (RuntimeException err) { + pendingWork.decrementAndGet(); + // The pool rejected it (shutting down). Closing is the honest answer; + // holding the connection open would promise service that is not coming. + drop(fd); + } + } + + /** + * The only place a served connection is closed, so the count stays honest. + * + * Idempotent, and it has to be: the stop() deadline closes what is still open + * while a worker may be using that same connection, and that worker calls here + * again on its next failed read. Closing twice decrements the count a second + * time and hands close() a descriptor number the OS may already have reused for + * something else, so the second call would shut down unrelated I/O. Winning the + * removal is what decides which call owns the teardown. + */ + private void drop(int fd) { + if(liveConnections.remove(new Integer(fd)) == null) { + return; + } + // Before anything else: a descriptor number is reused as soon as it is + // closed, so an entry left behind here would time out the NEXT connection + // to be handed that number. + pooledDeadlines.remove(new Integer(fd)); + Object h2 = http2Sessions.remove(new Integer(fd)); + if(h2 != null) { + ((Http2)h2).close(); + } + Object session = sessions.remove(new Integer(fd)); + if(session != null) { + Tls.closeSession(((Long)session).longValue()); + } + ServerSocket.closeFd(fd); + openConnections.decrementAndGet(); + } + + /** 0 when this connection is plaintext. */ + private long sessionOf(int fd) { + Object session = sessions.get(new Integer(fd)); + return session == null ? 0 : ((Long)session).longValue(); + } + + private static int readFrom(int fd, long session, byte[] buffer, int offset, int length) + throws IOException { + return session == 0 ? ServerSocket.read(fd, buffer, offset, length) + : Tls.read(session, buffer, offset, length); + } + + private static void writeTo(int fd, long session, byte[] buffer, int offset, int length) + throws IOException { + if(session == 0) { + ServerSocket.write(fd, buffer, offset, length); + } else { + Tls.write(session, buffer, offset, length); + } + } + + private void serve(int fd) { + trace("serve fd=" + fd); + activeRequests.incrementAndGet(); + try { + serveOne(fd); + } finally { + activeRequests.decrementAndGet(); + } + } + + /** A malformed request that deserves a specific status before the close. */ + private static final class ProtocolException extends IOException { + final int status; + + ProtocolException(int status, String message) { + super(message); + this.status = status; + } + } + + /** + * A connection plus whatever has been read from it and not yet consumed. + * + * The leftover is the point. A client may send a second request before reading + * the reply to the first, and both arrive in one read; a parser that keeps only + * the request it wanted silently drops the rest. That is not an exotic case -- + * it is what pipelining is, and what a proxy does when it coalesces. + */ + private final class Conn { + final int fd; + final long session; + byte[] buffer = new byte[0]; + int pos; + /** True while `buffer` is the thread's shared buffer rather than ours. */ + boolean borrowed; + /** + * True once this request's header slices name positions in `buffer`. + * + * "Is anything still pointing at this buffer" is the question fill() has to + * answer before it lets go of a borrow, and "is there anything left to + * read" is NOT the same question -- see the comment there. + */ + boolean parsedFromBuffer; + + /** Memoised request targets for this connection. See internTarget. */ + private final String[] targetCache = new String[TARGET_CACHE_SLOTS]; + + String internTarget(byte[] data, int start, int length) { + if(targetCache.length == 0) { + // Cache disabled (CN1_HTTP_TARGET_CACHE=0), for A/B measurement. + // Guarded because the slot arithmetic below is a modulo, and a zero + // size would divide by it rather than politely doing nothing. + return asciiString(data, start, length); + } + int hash = 0; + for(int iter = 0 ; iter < length ; iter++) { + hash = hash * 31 + data[start + iter]; + } + int slot = (hash & 0x7fffffff) % TARGET_CACHE_SLOTS; + String cached = targetCache[slot]; + if(cached != null && cached.length() == length) { + int iter = 0; + while(iter < length + && cached.charAt(iter) == (char)(data[start + iter] & 0xff)) { + iter++; + } + if(iter == length) { + return cached; + } + } + String fresh = asciiString(data, start, length); + // One entry per slot, overwritten on collision rather than chained: + // two hot targets that collide would otherwise both miss forever, and + // overwriting lets whichever is currently hot keep the slot. + targetCache[slot] = fresh; + return fresh; + } + /** + * Set when a read returned end-of-stream. The linger above has to tell a + * client that WENT AWAY from one that has merely gone quiet: the first + * must be closed, and handing the second to the poller is the whole point. + */ + boolean closedByPeer; + + /** + * Where a response is assembled, reused for the life of the connection. + * + * The head used to be built with a StringBuilder, turned into a String and + * then encoded to bytes, and the body copied in after that -- four + * allocations per response, and the StringBuilder reallocating its char[] + * as it grew. An allocation census put char[] at 47% of ALL allocation in + * this server, and this path was most of it. Bytes go in directly now: + * the header field names are ASCII constants, and a status or a length is + * digits. + */ + byte[] out = new byte[1024]; + int outLength; + /** + * Reused header slices: nameStart, nameLength, valueStart, valueLength. + * + * Handed to a Request BY REFERENCE, not copied -- and that is only safe + * because `buffer` is a fresh, exactly-sized array per fill, so each + * Request's `raw` is privately owned and immutable once parsed. The two are + * a pair: the slices name absolute offsets into that particular array. + * + * Anything that makes the buffer REUSABLE breaks the pair. Measured, with a + * capacity-plus-limit buffer in place of the per-fill array: a Request came + * back holding `raw.length=45` while its own slices named offsets 212 and + * 232, i.e. raw from one request and the slice table from a larger later + * one -- the shared table had been re-parsed under a Request still using it. + * The result was an ArrayIndexOutOfBoundsException in getHeader, thrown + * outside any try block, which killed the connection with no response + * written (~1 suite run in 2). + * + * TWO WRONG ANSWERS, so that a third attempt does not re-buy them. It is + * NOT two workers on one connection: a probe that reports a second thread + * entering serve() for a descriptor already inside it fired ZERO times on + * the build that fails (the same probe on the passing build proves nothing, + * which is how it was nearly mis-read). And it is not the slice table + * overflowing: slices.length was 64 against a headerCount of 3. + * + * THE ACTUAL CAUSE, and it is a property of the VM rather than of this + * class: a zero-copy buffer's LENGTH IS NOT STABLE. readIntoThreadBufferImpl + * hands back the same array object every call and mutates it in place -- + * `a->length = (int)n` -- because a ParparVM array's length is a field in a + * struct the runtime owns. So the array reports ~100 bytes while its headers + * are parsed (slices at 39, 59, 69) and reports 40 after the same thread's + * next read, with the already-parsed slices left naming positions past the + * end. That is the 40-against-69 reading, and nothing moved: the length did. + * + * `available()` is written as `buffer.length - pos` for exactly this reason. + * It re-reads the length every time and therefore self-corrects. Caching it + * in a `limit` field -- which is what a reusable buffer needs -- is what + * breaks, and it breaks silently, as a truncated response rather than a + * wrong one. + * + * So a reusable buffer has to stop borrowing first: take a private array + * (whose length really is immutable) before anything caches a length or + * parses slices out of it. Sizing that copy is itself subject to the same + * trap, since buffer.length must be read before the next read mutates it. + * + * THAT WAS BUILT, AND IT IS NOT WORTH IT. With the length handled correctly + * the reusable buffer is correct -- 4 default plus 2 virtual-thread suite + * runs clean, against a naive version that failed within two -- and it buys + * NOTHING. Measured in virtual-thread mode, same 12s window and load: + * + * cycles per window reuse 40, 40, 38 no reuse 41, 42, 41 + * requests 2.69M, 2.63M, 2.56M 3.08M, 3.01M, 2.70M + * + * The collection RATE does not move, so this array is not a meaningful part + * of the ~340 bytes a request allocates, and the throughput came out lower + * in all three reps (arms were not interleaved, so treat that half loosely). + * The per-request read buffer is simply not where the allocation is: look + * for the bytes before removing an allocation on the assumption it matters. + * + * So removing the per-request byte[] is not just a capacity field: a Request + * has to own a consistent (raw, slices, headerCount) triple, re-based + * together or not at all. + */ + int[] slices = new int[64]; + /** + * Where a deferred JSON body is serialised, so its length is known before + * the head that must declare it is written. Reused like everything else + * here; the copy into the head buffer afterwards is a memcpy of a body + * small enough to share a packet with its headers. + */ + final ByteSink bodySink = new ByteSink(512); + + void reset() { + outLength = 0; + } + + void ensure(int extra) { + if(outLength + extra <= out.length) { + return; + } + int size = out.length * 2; + while(size < outLength + extra) { + size *= 2; + } + byte[] grown = new byte[size]; + System.arraycopy(out, 0, grown, 0, outLength); + out = grown; + } + + /** ASCII only. Every caller passes a header name or a constant. */ + void put(String ascii) { + int n = ascii.length(); + ensure(n); + for(int iter = 0 ; iter < n ; iter++) { + out[outLength++] = (byte)ascii.charAt(iter); + } + } + + /** + * The content type, encoded once per connection rather than per response. + * + * put(String) walks charAt by charAt, and a handler hands back the same + * String instance every time -- a literal, or a constant on Response -- + * so after the first response the bytes are already there. Identity, not + * equals: a handler that builds a fresh String per response simply keeps + * missing and pays what it paid before, and the cache is filled ONCE so + * that case cannot allocate per request either. + */ + private String ctKey; + private byte[] ctBytes; + + void putContentType(String ct) { + if(ct == ctKey) { + System.arraycopy(ctBytes, 0, out, ensureAt(ctBytes.length), ctBytes.length); + outLength += ctBytes.length; + return; + } + put(ct); + if(ctKey == null && ct != null) { + ctKey = ct; + ctBytes = asciiBytes(ct); + } + } + + /** Reserves {@code n} bytes and answers the offset they start at. */ + private int ensureAt(int n) { + ensure(n); + return outLength; + } + + void put(byte[] data, int offset, int length) { + ensure(length); + System.arraycopy(data, offset, out, outLength, length); + outLength += length; + } + + void put(int b) { + ensure(1); + out[outLength++] = (byte)b; + } + + /** + * A non-negative number as ASCII digits, written in place. + * Integer.toString would allocate a String and its char[] -- per response, + * twice (the status and the content length). + */ + void putNumber(long value) { + if(value < 0) { + put("-"); + value = -value; + } + if(value == 0) { + put('0'); + return; + } + int start = outLength; + // Tried and REVERTED: an int fast path that sized the number with a + // ternary chain instead of dividing to count digits. It looked like a + // clear win on paper -- this was 2.1% of on-CPU time and a 64-bit + // divide is tens of cycles -- but measured 6 of 8 interleaved reps + // SLOWER, median about -3.7%. The likely reason is that a ten-way + // ternary compiles to branchy operand-stack code here, which costs more + // than the divisions it removes. Do not re-attempt without measuring on + // an idle machine; the reps above were taken at load 17 and are weak + // evidence, but there was no sign of a gain in any of them. + long v = value; + int digits = 0; + while(v > 0) { + digits++; + v /= 10; + } + ensure(digits); + outLength += digits; + int at = outLength; + while(value > 0) { + out[--at] = (byte)('0' + (int)(value % 10)); + value /= 10; + } + if(at != start) { + // Unreachable unless the digit count and the loop disagree; the + // buffer would be left with a hole rather than a short write. + throw new IllegalStateException("digit count mismatch"); + } + } + + Conn(int fd, long session) { + this.fd = fd; + this.session = session; + } + + int available() { + return buffer.length - pos; + } + + /** + * The one Response handed to Request.respond on this connection. Null until + * a handler asks for it, so a handler that never does pays nothing. + */ + Response pooledResponse; + + /** + * Requests answered on this connection since the last fold into the + * server's striped counter. Plain: one virtual thread owns a connection + * for its whole life, so this field has a single writer. + */ + long servedPending; + + /** Index into servedStripes for the host that owns this connection, or -1. */ + int stripe = -1; + + /** + * The one Request served on this connection, re-pointed per request rather + * than reallocated. Response and Request were the whole of what /plaintext + * still allocated once the borrowed-buffer copy went: 88 and 80 bytes, one + * of each, every request. + */ + Request pooledRequest; + + /** Reads more. False at end of stream. */ + boolean fill(byte[] scratch) throws IOException { + if(borrowed && available() == 0 && !parsedFromBuffer) { + // Nothing is left unread AND nothing has been parsed out of this + // buffer, so this is the START of a new request on a kept-alive + // connection and there is nothing to preserve: the previous request + // was answered before the loop came back here. Just let go. + // + // The parsedFromBuffer half is load-bearing and was missing. An + // empty buffer does NOT mean no request is in flight: a POST whose + // headers arrive in one segment and whose body arrives in the next + // reaches the body loop with the header block fully consumed, so + // available() is 0 while the Request's slices still name positions + // in this very buffer. Taking this branch there dropped the borrow, + // skipped detachPreservingOffsets below, and let the next zero-copy + // read overwrite the headers with the body -- after which + // getHeader("connection") walked off the end of the array and the + // AIOOBE killed the connection with no response written at all. + // + // That is a truncated reply, not a wrong one, so it showed up only + // as a client-side timeout: transactionRollsBack failing 15.05s + // (its own setSoTimeout) with status -1, and authGuardsMutatingRoutes + // reading an empty body, about 2 full-suite runs in 6. It never + // appeared under virtual threads because ZERO_COPY_READ is off + // there, which is also why it survived the whole reactor rewrite. + // + // Copying here instead is what a first version did, and it put the + // per-request byte[] straight back -- the linger means a worker + // loops without handing the descriptor back, so `borrowed` was still + // set on every subsequent request and each one copied the whole + // buffer. The census said 223 bytes per request where it should have + // said none, which is the only reason it was noticed. + buffer = EMPTY_BODY; + pos = 0; + borrowed = false; + } else if(borrowed) { + // A second read WITHIN one request is about to overwrite the + // thread buffer, and a Request parsed out of it holds SLICES into + // exactly that memory. Copy first, preserving absolute offsets so + // those slices stay valid. + // + // Found by BackendHttpIntegrationTest.transactionRollsBack, which + // sends a body big enough to need two reads along with an + // Authorization header: the second read landed on top of the header + // and the request came back 401 instead of 400. A corrupted header + // is the good version of this bug -- the same overwrite could just + // as easily have served one request's bytes inside another's. + detachPreservingOffsets(); + } + if(ZERO_COPY_READ && available() == 0 && session == 0) { + // The common case by far: nothing left over, so the bytes are read + // into this thread's reusable buffer and parsed where they land -- + // no array allocated, nothing copied. The request is parsed out of + // the same memory the kernel wrote into. + // + // The returned array's length is exactly what was read, so every + // parser below that scans to buffer.length keeps working untouched. + // That is only possible because the length of a ParparVM array is a + // field in a struct we own; introducing a separate limit instead + // would have meant auditing fifteen call sites, and one missed site + // reads a previous request's bytes into this one's response. + // + // Plaintext only (session == 0): a TLS read decrypts through its + // own path and does not hand back a buffer we own. + byte[] direct = ServerSocket.readIntoThreadBuffer(fd, scratch.length); + if(direct == null) { + closedByPeer = true; + return false; + } + if(direct.length == 0) { + // Nothing ready on a non-blocking descriptor, which means two + // different things and only one of them is trouble. + // + // MIDWAY THROUGH a message it is not the peer leaving: the + // headers arrived in one packet and the body is still coming, + // and answering "closed" here dropped a valid upload. Those + // reads go to the copying path, which parks on EAGAIN and owns + // its buffer -- this one cannot park, because the storage is + // per HOST thread and another virtual thread's read would + // overwrite what this one is about to return. + // + // BETWEEN messages it is the ordinary quiet of a kept-alive + // connection, and reporting it as closed is how a worker is + // freed. Sending those to a parking read instead made the + // suite 2.4x slower and stopped shedsIdleConnections shedding + // anything -- the partial request sat until a deadline and was + // answered 408 rather than dropped. + // + // parsedFromBuffer is precisely that distinction, and it is + // already maintained for fill()'s benefit. Note a SPLIT header + // block needs nothing here: after the first partial read + // available() is non-zero, so it never takes this branch. + if(!parsedFromBuffer) { + closedByPeer = true; + return false; + } + return fillCopying(scratch); + } + if(ZERO_COPY_MODE == 2) { + // Diagnostic bisection only -- see ZERO_COPY_MODE. Same read as + // mode 1, same heap array as mode 0, so whichever of the two the + // throughput follows is the one that costs. + byte[] owned = new byte[direct.length]; + System.arraycopy(direct, 0, owned, 0, direct.length); + buffer = owned; + pos = 0; + borrowed = false; + return true; + } + buffer = direct; + pos = 0; + borrowed = true; + return true; + } + return fillCopying(scratch); + } + + /** + * The copying read: into this connection's own scratch, then into a buffer + * sized for what is kept plus what arrived. + * + * Split out of fill() so the zero-copy path can defer to it when the + * descriptor has nothing ready. readFrom parks on EAGAIN for a virtual + * thread, which is the behaviour the shared-buffer read cannot safely have. + */ + private boolean fillCopying(byte[] scratch) throws IOException { + int n = readFrom(fd, session, scratch, 0, scratch.length); + if(n <= 0) { + closedByPeer = true; + return false; + } + int keep = available(); + byte[] grown = new byte[keep + n]; + System.arraycopy(buffer, pos, grown, 0, keep); + System.arraycopy(scratch, 0, grown, keep, n); + buffer = grown; + pos = 0; + borrowed = false; + return true; + } + + /** + * Reads until `needed` bytes are buffered, into ONE array sized for them. + * + * fill() grows by exactly what it just read, so a body arriving in + * scratch-sized pieces reallocated and recopied everything once per read: + * an 8MB upload over an 8KB buffer is about a thousand resizes and some 4GB + * of copying before the handler is even called, which a few concurrent + * uploads turn into the whole machine. + * + * When the total is known -- and for Content-Length it is -- the destination + * can be grown toward it in doublings, which is one copy of what was already + * buffered and an amortised one of the body. + * + * It is NOT allocated at `needed` up front, which is what this did first. + * Content-Length is a client's CLAIM, and believing it before a byte of the + * body has arrived means an unauthenticated client can make the server + * allocate 8MB by sending a header and then nothing at all: this loop holds + * that memory until the rate allowance below expires, and the connection + * ceiling is in the thousands, so a few dozen such requests are gigabytes. + * Growing as the bytes ARRIVE makes the memory track what was actually sent, + * which is the only figure a client cannot lie about. The doubling is what + * keeps that affordable -- growing by each read's size instead was the + * original defect here, about a thousand resizes and 4GB of copying for one + * 8MB upload. + * + * The invariant the rest of this class depends on is kept, because every + * growth is capped at `needed`: the last one allocates exactly that, so the + * array handed over is exactly `needed` long with every byte valid, and + * `buffer.length` still means "bytes readable". See the class comment for + * why a `limit` field is not the answer here. + */ + boolean fillTo(int needed) throws IOException { + int keep = available(); + if(keep >= needed) { + return true; + } + long charged = 0; + try { + // RESERVED before allocated, not after. The charge is what bounds + // concurrent uploads, and a budget checked after the allocation + // bounds nothing: every thread that reaches a growth boundary at the + // same moment takes its memory first and finds out it was over the + // limit second, so the peak is the number of threads times their + // step, whatever the limit says. Reserving first makes the refusal + // happen while the memory is still hypothetical. `charged` is + // incremented in the same breath, so the finally below rolls the + // reservation back even if the allocation itself fails. + int first = Math.max(keep, Math.min(needed, BODY_CHUNK_BYTES)); + charged += first; + if(http1UploadBytes.addAndGet(first) > MAX_HTTP1_UPLOAD_BYTES) { + throw new ProtocolException(503, "too many uploads in flight"); + } + byte[] grown = new byte[first]; + System.arraycopy(buffer, pos, grown, 0, keep); + int at = keep; + // A RATE, not a deadline. The head gets a flat bound because it is small; + // a body cannot, since 8 MiB over a slow mobile link is a real client and + // any fixed wall-clock limit refuses it. But SO_RCVTIMEO restarts on + // every successful read, so without something here a client declaring a + // large Content-Length and sending one byte inside each window holds its + // worker for as long as it likes -- and in pool mode, which is what TLS + // uses, enough of those are the whole server. The allowance is what this + // many bytes take at the floor rate, plus one socket timeout of slack, so + // a slow upload that keeps making progress finishes and a dribble does not. + long started = System.currentTimeMillis(); + long allowed = SOCKET_TIMEOUT_MILLIS + + (long)(needed - keep) * 1000L / MIN_BODY_BYTES_PER_SECOND; + while(at < needed) { + if(System.currentTimeMillis() - started > allowed) { + throw new ProtocolException(408, "the request body did not arrive in time"); + } + if(at == grown.length) { + // Doubling, capped at what was declared -- so the final growth + // lands exactly on `needed` and the invariant above holds. + int next = (int)Math.min((long)needed, (long)grown.length * 2); + // Reserved before allocated, for the reason above. + long delta$ = (long)next - grown.length; + charged += delta$; + if(http1UploadBytes.addAndGet(delta$) > MAX_HTTP1_UPLOAD_BYTES) { + throw new ProtocolException(503, "too many uploads in flight"); + } + byte[] bigger = new byte[next]; + System.arraycopy(grown, 0, bigger, 0, at); + grown = bigger; + } + // Exactly the shortfall, so a pipelined request behind this body stays + // in the socket for the next parse rather than being read into it. + int n = readFrom(fd, session, grown, at, grown.length - at); + if(n <= 0) { + closedByPeer = true; + return false; + } + at += n; + } + buffer = grown; + pos = 0; + borrowed = false; + return true; + } finally { + // Every path out: the body arrived, the peer went away, the + // deadline passed, or the process was full. The charge covers + // the READ -- on success the buffer becomes the connection's and + // the request goes on to a handler, which is ordinary server + // memory rather than an upload being held open. + http1UploadBytes.addAndGet(-charged); + } + } + + /** + * Give up the shared thread buffer before this connection can be taken by a + * different worker. + * + * The buffer belongs to the THREAD, not the connection. Anything still + * unread has to be copied somewhere this connection owns before the + * descriptor goes back to the reactor: the next worker runs on another + * thread whose buffer is different memory, and the thread that read these + * bytes overwrites them on its next request. + * + * The copy happens only when bytes are actually left over -- the pipelining + * case. The ordinary request-per-read path copies nothing. + */ + /** + * Take a private copy of the whole borrowed buffer, keeping every index the + * same, so anything already parsed out of it (a Request's header slices) + * keeps pointing at the right bytes. + * + * Compacting here instead would be a subtle disaster: it moves the content + * to offset zero while the slices still name the old positions. + */ + void detachPreservingOffsets() { + byte[] owned = new byte[buffer.length]; + System.arraycopy(buffer, 0, owned, 0, buffer.length); + buffer = owned; + borrowed = false; + } + + void releaseBorrowed() { + if(!borrowed) { + return; + } + int keep = available(); + if(keep > 0) { + byte[] owned = new byte[keep]; + System.arraycopy(buffer, pos, owned, 0, keep); + buffer = owned; + } else { + buffer = EMPTY_BODY; + } + pos = 0; + borrowed = false; + } + + void write(byte[] data) throws IOException { + writeTo(fd, session, data, 0, data.length); + } + } + + private void serveOne(int fd) { + long session; + try { + // A POOL worker owns its descriptor and blocks on it: there is no one to + // hand its host thread to. A virtual thread is the opposite, and blocking + // here defeated the whole design -- readImpl reaches its park path only + // when recv returns EAGAIN, which a blocking descriptor never does. So the + // virtual thread never parked and the host OS thread sat in the kernel + // until SO_RCVTIMEO. A load generator never shows this, because the bytes + // are always already there; a client sending one byte per timeout pins a + // host and starves every connection scheduled on it. + // + // TLS is the exception, and stays blocking below: Tls.readImpl maps + // SSL_ERROR_WANT_READ to a hard error rather than parking, so a + // non-blocking descriptor would break TLS reads outright. Giving the TLS + // layer a park path is the real fix and is not this change. + boolean parking = virtualThreads; + if(!parking) { + ServerSocket.setBlocking(fd, true); + } + if(tls != null && sessionOf(fd) == 0) { + // The handshake runs here, on the worker, because the descriptor is + // blocking here and a handshake is several round trips. On the + // reactor thread it would stall every other connection. + long fresh = tls.accept(fd); + if(fresh == 0) { + // Not a TLS client, or no common cipher. Ordinary traffic. + drop(fd); + return; + } + sessions.put(new Integer(fd), new Long(fresh)); + } + session = sessionOf(fd); + if(tls != null && Http2.ALPN.equals(Tls.negotiatedProtocol(session))) { + // ALPN settled on h2, so this connection is framed, not textual, + // for its whole life. There is no downgrade from here. + serveHttp2(fd, session, null, 0); + return; + } + } catch (Exception err) { + drop(fd); + return; + } + + Conn conn = new Conn(fd, session); + // Which stripe this connection's requests count into. Resolved once here + // rather than per request: the owner cannot change for a live descriptor. + if(virtualThreads && fd >= 0 && fd < vtOwnerByFd.length && servedStripes.length > 0) { + int host = vtOwnerByFd[fd]; + if(host >= 0 && host * SERVED_STRIPE_STRIDE < servedStripes.length) { + conn.stripe = host * SERVED_STRIPE_STRIDE; + } + } + byte[] scratch = new byte[8192]; + int served = 0; + + // Cleartext HTTP/2 by prior knowledge: a client that already knows the + // server speaks h2 opens with the connection preface instead of a request + // line. This is how gRPC talks over cleartext and how a load balancer that + // terminated TLS talks to an origin, and it is the only way to reach h2 + // without ALPN. + if(http2Sessions.containsKey(new Integer(fd))) { + serveHttp2(fd, session, null, 0); + return; + } + try { + while(conn.available() < HTTP2_PREFACE.length) { + if(!conn.fill(scratch)) { + drop(fd); + return; + } + if(!startsWithPrefacePrefix(conn)) { + break; // definitely not h2; parse it as HTTP/1.1 + } + } + if(conn.available() >= HTTP2_PREFACE.length && matchesPreface(conn)) { + byte[] rest = new byte[conn.available()]; + System.arraycopy(conn.buffer, conn.pos, rest, 0, rest.length); + serveHttp2(fd, session, rest, rest.length); + return; + } + } catch (Exception err) { + drop(fd); + return; + } + + while(true) { + Request request; + try { + request = readRequest(conn, scratch); + } catch (ProtocolException err) { + trace("fd=" + fd + " rejected: " + err.getMessage()); + writeStatusOnly(conn, err.status, err.getMessage()); + drop(fd); + return; + } catch (ServerSocket.TimeoutException err) { + // An idle client, not a fault. Shedding it is the point of the deadline. + trace("fd=" + fd + " timed out"); + drop(fd); + return; + } catch (Exception err) { + trace("fd=" + fd + " read failed: " + err); + drop(fd); + return; + } + if(request == null) { + drop(fd); // the peer closed + return; + } + + boolean keepAlive = wantsKeepAlive(request); + // Methods are case-sensitive, so this is an exact comparison. + boolean headOnly = "HEAD".equals(request.getMethod()); + Response response; + // From here to the end of the write is the request being in flight. Not + // the whole of serveOne: that is the CONNECTION, which outlives this. + inFlightRequests.incrementAndGet(); + try { + try { + response = handler.handle(request); + if(response == null) { + response = Response.text(404, "not found"); + } + } catch (Exception err) { + System.err.println("handler failed: " + err); + response = Response.text(500, "internal error"); + } + try { + writeResponse(conn, fd, session, response, keepAlive, headOnly); + if(conn.stripe >= 0) { + servedStripes[conn.stripe]++; // single writer: this host + } else { + requestsServed.incrementAndGet(); // reactor mode, no stripes + } + } catch (Exception err) { + trace("fd=" + fd + " write failed: " + err); + drop(fd); + return; + } + } finally { + inFlightRequests.decrementAndGet(); + } + if(!keepAlive) { + drop(fd); + return; + } + if(conn.available() > 0) { + continue; // a pipelined request is already in the buffer + } + // The burst cap is a fairness backstop for a POOL: it stops one worker + // monopolising a shared thread. A virtual thread owns its connection + // and parking costs nobody anything, so there is nothing to be fair + // to -- and breaking here would be worse than pointless, because in + // this mode the loop's exit path closes the connection. That is a + // healthy keep-alive connection dropped every 256 requests, which the + // client sees as a mid-stream close: 446833 write errors against + // 164307 requests at four connections, and it made every earlier + // virtual-thread measurement an underestimate. + if(++served >= KEEPALIVE_BURST_LIMIT) { + if(virtualThreads) { + // Step aside rather than close. A virtual thread under a load + // generator never runs out of bytes, so it never parks on its + // own and would hold this host thread for as long as the + // client kept talking -- with fewer hosts than connections the + // rest starve, measured as two hosts serving two of sixty four. + // Breaking here instead is worse still, because in this mode + // the exit path CLOSES the connection: 446833 write errors + // against 164307 requests, a healthy keep-alive connection + // dropped every 256 requests. + served = 0; + conn.releaseBorrowed(); + VirtualThread.yieldNow(); + continue; + } + break; // fairness backstop; see the constant + } + // On a virtual thread there is nothing to be fair TO: parking releases + // the host thread immediately, so holding the connection costs no one + // anything and handing it back would only add a poller round trip per + // request. + if(!virtualThreads && pendingWork.get() > 0 + && workerCount - activeRequests.get() <= pendingWork.get()) { + // Hand back only when something is actually waiting AND there are + // not enough idle workers for it -- the case where holding this + // connection denies service to another. With nothing waiting, or + // with a spare worker for whoever is, holding costs nobody + // anything. Testing the idle count alone is wrong in the most + // common configuration of all: with connections == workers every + // worker is busy and nothing is queued, so "idle <= pending" reads + // 0 <= 0 and hands the connection back on every single request, + // which is the behaviour the linger exists to avoid. + // + // Breaking on "anything is waiting at all" was the first attempt and + // it is too blunt: with 128 workers and 64 connections every worker + // handed its connection back on every request even though half the + // pool was idle, and throughput did not move (123k at 16 workers, + // 126k at 128). The pool size only buys anything if a spare worker + // actually lets a connection stay put. + break; + } + // Wait briefly for the next request rather than going round the poller + // for it. See KEEPALIVE_LINGER_MILLIS. + // + // With the workers polling, the linger waits ZERO milliseconds: it + // still asks whether the next request has already arrived, because + // answering a pipelined request on the spot is free, but it never + // BLOCKS waiting for one. Two reasons, and they are the reasons the + // linger exists at all: + // + // - What it buys is avoiding the handback, and in this mode the + // handback is one epoll_ctl with no wake and no queue. There is + // almost nothing left to avoid. + // - What it costs is much higher here. A lingering worker is not + // polling, so with fewer workers than connections it withholds the + // poller itself. In the dispatching mode a lingering worker only + // withheld itself, because a separate thread went on polling. + // + // This is what Go does: read optimistically, park on EAGAIN + // (internal/poll.FD.Read). The park is what re-arming is here. + // -1 on a virtual thread: wait for the next request for as long as the + // client cares to take. That is not a blocked thread, it is a parked + // virtual thread costing a stack and nothing else, which is exactly + // the resource an idle keep-alive connection should cost. + int linger = virtualThreads ? -1 : KEEPALIVE_LINGER_MILLIS; + if(virtualThreads || linger > 0) { + boolean more; + try { + // A readiness wait rather than a timed read: it is one syscall + // and it leaves the receive deadline alone, so the request this + // is waiting for still gets the full one when it arrives. + // Setting and restoring SO_RCVTIMEO around each wait did work, + // and cost four setsockopt per request -- 15% of syscall time. + if(!ServerSocket.awaitReadable(fd, linger)) { + break; // quiet client; the poller can have it + } + // The request this buffer was parsed from has been ANSWERED, + // so nothing points into it any more and the borrow can be + // dropped rather than copied. + // + // Without this, fill() below sees parsedFromBuffer still set + // from the request just served, reads that as "midway through a + // request", and takes detachPreservingOffsets -- a full copy of + // the borrowed buffer on EVERY keep-alive request. Measured on + // /plaintext under virtual threads: 1989689 detaches against + // 2000000 reads, one 97-byte array per request, 37% of + // everything the route allocated. The zero-copy read was + // working perfectly and handing the saving straight back here. + // + // The flag's real job is the SECOND read within one request (a + // body arriving after its headers), where slices into this + // array are live and the copy is required. That case is + // untouched: readRequest clears the flag on entry and raises it + // once the header block is parsed, so it is set exactly across + // the window where a Request exists. This point is outside that + // window by construction -- the handler has returned and the + // response is on the wire. + conn.parsedFromBuffer = false; + more = conn.fill(scratch); + } catch (IOException err) { + drop(fd); + return; + } + if(more) { + continue; + } + // Readable but nothing came: the peer closed. + drop(fd); + return; + } + break; + } + // Before the descriptor can be taken by another worker: the read buffer + // belongs to THIS thread and the next request on it will overwrite these + // bytes. Must come before reactor.add, not after -- the moment the fd is + // registered, another worker can pick it up. + conn.releaseBorrowed(); + try { + // Back to the poller for the next request on this connection. Both + // epoll_ctl and kevent are safe to call from this thread. + if(virtualThreads) { + // Reached only when the connection itself is finished: a virtual + // thread does not come back here to wait, it parks where it waits. + // Re-arming now would hand the poller a descriptor nobody owns. + drop(fd); + } else { + ServerSocket.setBlocking(fd, false); + armConnection(fd, false); + } + } catch (IOException err) { + drop(fd); + } + } + + /** + * One turn of an HTTP/2 connection: read what is available, answer every + * request that completed, flush, and hand the descriptor back to the reactor. + * + * Deliberately the same shape as the HTTP/1.1 path rather than a worker that + * owns the connection for its lifetime. h2 connections are long-lived by + * design, so pinning a worker to each would mean the pool size is the limit on + * concurrent clients -- the exact thing the reactor exists to avoid. + */ + private void serveHttp2(int fd, long session, byte[] pending, int pendingLength) { + Http2 h2; + // Held for the WHOLE turn, not just while a handler runs. The per-stream + // count below drops to zero as the last response is submitted, and the + // flush and the liveness check after the loop still call into nghttp2 and + // the TLS session -- so stop() could see nothing in flight and free both + // underneath this thread. A turn with no completed request at all, one + // that only pumped control frames, was never counted by anything. + http2Turns.incrementAndGet(); + try { + Object existing = http2Sessions.get(new Integer(fd)); + if(existing == null) { + // Told to the native side once, where the reservation happens. + // Idempotent, so doing it per session rather than finding a + // startup hook costs an atomic store on a path that is already + // creating a session. + Http2.setMaxBodyBytes(MAX_OPEN_H2_BODY_BYTES); + Http2.setMaxFileBodies(MAX_OPEN_H2_FILES); + h2 = Http2.create(); + http2Sessions.put(new Integer(fd), h2); + // The SETTINGS preface has to reach the client before anything else. + flushHttp2(fd, session, h2); + } else { + h2 = (Http2)existing; + } + + if(pending != null && pendingLength > 0) { + // Bytes already read while deciding this was h2, preface included. + h2.receive(pending, 0, pendingLength); + } else { + byte[] scratch = new byte[16384]; + int n = readFrom(fd, session, scratch, 0, scratch.length); + if(n <= 0) { + drop(fd); + return; + } + h2.receive(scratch, 0, n); + } + + // Every submitted body is COPIED into a native buffer that lives until + // the flush after this loop, so a connection completing many streams at + // once holds all of them at the same time: with the concurrency this + // server advertises and an endpoint returning a large body, one client + // could hold hundreds of megabytes of native response buffers on top of + // the Java ones. Draining when enough has piled up bounds that without + // paying a syscall per response. + long queuedBodyBytes = 0; + Http2.Stream stream; + while((stream = h2.nextRequest()) != null) { + // :authority is what Host is in HTTP/1.1, so the handler sees a + // request shaped exactly like an HTTP/1.1 one. + Map headers = new LinkedHashMap(stream.getHeaders()); + if(stream.getAuthority() != null) { + headers.put("host", stream.getAuthority()); + } + byte[] h2RequestBody = stream.getBody(); + if(h2RequestBody != null && h2RequestBody.length > 0 + && !Utf8.isValid(h2RequestBody, 0, h2RequestBody.length)) { + // Decided here rather than in getBodyAsString, because this is + // where a status code can be produced: the decoder has no way + // to answer 400, and returning null there would have made a + // malformed body indistinguishable from an absent one. + if(!h2.respond(stream.getId(), 400, "text/plain", new ArrayList(), + asciiBytes("the request body is not valid UTF-8"))) { + // The explanation is itself a body, and under a full + // process budget respond() takes nothing and says so. This + // path ignored that and moved on, so the stream was left + // unanswered until the connection timed out -- a client + // that sent bad bytes under load simply hung. The status + // still has to arrive; only the sentence is optional. + h2.respond(stream.getId(), 400, "text/plain", new ArrayList(), null); + } + requestsServed.incrementAndGet(); + continue; + } + Request request = new Request(stream.getMethod(), stream.getPath(), + "HTTP/2", headers, stream.getBodyAsString()); + Response response; + inFlightRequests.incrementAndGet(); + try { + response = handler.handle(request); + if(response == null) { + response = Response.text(404, "not found"); + } + } catch (Exception err) { + System.err.println("handler failed: " + err); + response = Response.text(500, "internal error"); + } + try { + boolean headOnly = "HEAD".equals(stream.getMethod()); + List extra = new java.util.ArrayList(); + // RFC 9110 6.6.1: an origin server with a clock MUST send Date, and + // the HTTP/1 writer does. This path sent only the content type and + // whatever the handler added -- and a handler cannot make up for it, + // because "date" is refused as server-owned. Caches were left without + // the timestamp they compute freshness and age from. + // ONE entry, and a complete line: Http2.headerLines() treats every + // element as "name: value" and the native parser drops anything + // without a colon. Added as two elements this produced two lines it + // ignored, so the header was still absent and nothing failed -- no + // test asserted it, which is why the first attempt looked right. + extra.add("date: " + currentHttpDate()); + if(response.extraHeaders != null) { + java.util.Iterator it = response.extraHeaders.keySet().iterator(); + while(it.hasNext()) { + Object key = it.next(); + Object value = response.extraHeaders.get(key); + if(key != null && value != null) { + String name = String.valueOf(key); + String text = String.valueOf(value); + // The native side splits this block on '\n', so a newline + // here is another field exactly as it is over HTTP/1.1. + if(isServerOwnedHeader(name)) { + System.err.println("dropped a response header the " + + "server owns: " + sanitizeForLog(name)); + } else if(isHeaderName(name) && isHeaderSafe(text)) { + extra.add(name + ": " + text); + } else { + System.err.println("dropped a response header whose name " + + "is not a token or whose value carries a control " + + "character: " + sanitizeForLog(name)); + } + } + } + } + String contentType = safeContentType(response.contentType); + // The same rule as HTTP/1: a 204, 304 or 1xx carries no body, so + // a DATA frame must not follow the headers here either. + boolean noBody = headOnly || statusForbidsBody(response.status); + if(response.fileFd >= 0 && !noBody + && Http2.pendingBodyFiles() >= MAX_OPEN_H2_FILES) { + // BEFORE submitting, not after. The turn check below stops + // this session, but every other session wakes on a control + // frame and submits one more first, so the cap was really + // "the cap plus one per connection" -- and a peer holding its + // window shut can keep waking them. Descriptors are a process + // resource and running out stops the server accepting sockets + // at all, which is a failure for every client rather than the + // one that caused it. + // + // Answered rather than deferred: the handler has ALREADY + // opened the descriptor, so holding the response holds the + // very thing being rationed. Closing it and saying so is the + // honest answer, and 503 is what it is. + StaticFiles.closeFile(response.fileFd); + // Even this small explanation is a body, and a body is what + // the ceiling refuses. If there is no room for it, the status + // alone still has to reach the client -- dropping the whole + // response would leave the stream hanging. + if(!h2.respond(stream.getId(), 503, "text/plain", extra, + asciiBytes("too many files in flight"))) { + h2.respond(stream.getId(), 503, "text/plain", extra, null); + } + } else if(response.fileFd >= 0 && !noBody) { + // Streamed frame by frame out of the descriptor. Reading the file + // in first cost its whole size in the heap plus the same again in + // the native copy, so a large enough public file turned one request + // into an OutOfMemoryError -- which the catch above does not catch, + // because it is an Error. The descriptor belongs to the session + // from here, so nothing on this side closes it. + if(h2.respondFile(stream.getId(), response.status, contentType, + extra, response.fileFd, response.fileOffset, response.fileLength)) { + // The session owns it from here and frees it natively. + StaticFiles.handOverFile(response.fileFd); + } else { + // Refused by the descriptor ceiling, which means the + // session took NOTHING -- the fd is still ours to close. + // The Java-side check above is now an early-out rather + // than the enforcement; this is the enforcement, and it + // happens in the same step that takes the descriptor. + StaticFiles.closeFile(response.fileFd); + h2.respond(stream.getId(), 503, "text/plain", extra, null); + } + } else { + // A HEAD describes the representation it is not sending, and + // that is the whole point of asking: over HTTP/1 this server + // reports the real length, so over HTTP/2 it has to as well, + // or the same static file answers a size on one protocol and + // nothing on the other from one handler. Only for a HEAD -- + // a bodiless STATUS has no representation to describe, which + // is the distinction the HTTP/1 writer already makes. + // ... and only where the status permits a length at all. A + // HEAD of a 204 must not carry one, which statusForbidsLength + // already knows and the HTTP/1 writer already honours -- so + // adding it here unconditionally made the SAME response valid + // over one protocol and invalid over the other, which is the + // exact divergence this fix existed to remove. + // The descriptor is NOT closed here, and a review that says + // it leaks is reading one branch short: responseBodyFor() + // below closes it in a finally, which is the entire reason it + // is called on a path that wants no body. Closing it here as + // well was measured at exactly one extra close per request -- + // openStaticFiles ran to -10 over ten HEADs -- and a double + // close is worse than the leak it was meant to fix, because + // the number is reusable the instant the first close returns + // and the second one then lands on whatever took it. + if(headOnly && !statusForbidsLength(response.status)) { + long described; + if(response.fileFd >= 0) { + described = response.fileLength; + } else if(response.hasDeferredJson) { + // respondJson leaves the value UNSERIALISED so the + // HTTP/1 writer can render it straight into the + // connection's buffer, which means response.body is + // empty and measuring it reports zero for a + // representation that is not. Rendering it is the only + // way to know the length, and describing the + // representation is the entire purpose of a HEAD. + described = responseBodyFor(response, false).length; + } else { + described = response.body == null ? 0 : response.body.length; + } + extra.add("content-length: " + described); + } + byte[] h2Body = responseBodyFor(response, noBody); + int bodyBytes = h2Body == null ? 0 : h2Body.length; + // The RESERVATION is the check. Testing the counter here and + // allocating inside respond() is two steps with a gap: two + // sessions being processed at once both read the total below + // the ceiling and then both allocate, so the real peak was the + // limit plus a body for every concurrent responder. respond() + // reserves and allocates in the same step natively, and + // answers false having taken nothing when the body would + // cross the ceiling. + if(!h2.respond(stream.getId(), response.status, contentType, extra, + h2Body)) { + // Bodiless, because the reason for refusing is that there + // is no room for bodies. An explanatory body here is the + // one allocation that must not be attempted. + h2.respond(stream.getId(), 503, "text/plain", extra, null); + } else { + queuedBodyBytes += bodyBytes; + } + } + requestsServed.incrementAndGet(); + if(queuedBodyBytes > MAX_QUEUED_H2_BODY_BYTES + || Http2.pendingBodyFiles() > MAX_OPEN_H2_FILES + || Http2.pendingBodyBytesAll() > MAX_OPEN_H2_BODY_BYTES) { + flushHttp2(fd, session, h2); + // What the flush could NOT write, not zero. nghttp2 pulls + // from a submitted body only as the peer's flow-control + // window allows, so a client that simply stops sending + // WINDOW_UPDATE makes every flush a no-op while the bodies + // stay retained. Zeroing a turn-local counter against that + // bounds nothing: the advertised stream concurrency times a + // large endpoint is hundreds of megabytes of native buffers + // held for a client that is reading none of it. + queuedBodyBytes = h2.pendingBodyBytes(); + if(queuedBodyBytes > MAX_QUEUED_H2_BODY_BYTES + || Http2.pendingBodyFiles() > MAX_OPEN_H2_FILES + || Http2.pendingBodyBytesAll() > MAX_OPEN_H2_BODY_BYTES) { + // Still over after a real attempt to write, so the peer + // is not draining. Leave the rest of the ready requests + // where they are -- their inbound bodies are already + // capped by the session limit -- and end the turn. The + // WINDOW_UPDATE that unblocks this connection wakes it + // again, and a peer that sends nothing at all is closed + // by the idle deadline rather than held forever. + break; + } + } + } finally { + // Held until the response has been SUBMITTED, not merely produced. + // Releasing it after the handler let stop() see no work in flight + // while this thread was still about to call into nghttp2 -- so the + // deadline sweep could close the descriptor and free the session + // underneath it, which truncates the response at best. + inFlightRequests.decrementAndGet(); + } + } + flushHttp2(fd, session, h2); + if(!h2.isAlive()) { + drop(fd); + return; + } + ServerSocket.setBlocking(fd, false); + armConnection(fd, false); + } catch (Exception err) { + trace("fd=" + fd + " http/2 failed: " + err); + drop(fd); + } finally { + http2Turns.decrementAndGet(); + } + } + + /** The HTTP/2 connection preface, sent by a client that opens with h2. */ + private static final byte[] HTTP2_PREFACE = prefaceBytes(); + + private static byte[] prefaceBytes() { + try { + return "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes("UTF-8"); + } catch (IOException err) { + return new byte[0]; + } + } + + /** + * True while what has arrived is still consistent with the preface. Lets the + * read loop stop early on an ordinary request rather than waiting for 24 bytes + * that will never match -- "GET / HTTP/1.1" diverges at the second character. + */ + private static boolean startsWithPrefacePrefix(Conn conn) { + int have = Math.min(conn.available(), HTTP2_PREFACE.length); + for(int iter = 0 ; iter < have ; iter++) { + if(conn.buffer[conn.pos + iter] != HTTP2_PREFACE[iter]) { + return false; + } + } + return true; + } + + private static boolean matchesPreface(Conn conn) { + for(int iter = 0 ; iter < HTTP2_PREFACE.length ; iter++) { + if(conn.buffer[conn.pos + iter] != HTTP2_PREFACE[iter]) { + return false; + } + } + return true; + } + + private void flushHttp2(int fd, long session, Http2 h2) throws IOException { + byte[] out = h2.drain(); + while(out != null && out.length > 0) { + writeTo(fd, session, out, 0, out.length); + out = h2.drain(); + } + } + + /** + * The response body as bytes. A file-backed response cannot use sendfile on an + * HTTP/2 connection -- the bytes have to become DATA frames, which means they + * have to be produced here -- so it is read in, and the descriptor is released + * either way. + */ + /** + * Whether this status ends the response at the header section. + * + * RFC 9110: a 1xx, 204 or 304 response carries no body, and a client stops + * reading at the blank line. Writing one anyway does not merely waste bytes + * -- on a keep-alive connection the client reads those bytes as the start of + * the NEXT response, and everything after that on the connection is + * misframed. Only HEAD used to be treated this way. + */ + static boolean statusForbidsBody(int status) { + // 205 belongs here with 204: RFC 9110 15.3.6 says a Reset Content + // response cannot contain content and is terminated by the first empty + // line, so a handler that returns bytes with it desynchronises a + // keep-alive connection exactly the way a 204 with bytes does. + return status == 204 || status == 205 || status == 304 + || (status >= 100 && status < 200); + } + + /** + * Whether this status must not carry Content-Length at all. + * + * RFC 9110 6.4.1 makes that a MUST NOT for 1xx and 204. + * + * 304 is here too, which is a correction. The rule for one is not that it + * carries no length but that any length it carries must describe the + * SELECTED REPRESENTATION -- what a 200 for the same request would have + * sent. Nothing here knows that: a 304 is built by StaticFiles as + * Response.empty, so the only figure available is zero, and sending + * "Content-Length: 0" tells the cache the file it just validated is empty. + * The header is optional on a 304, so omitting it is both correct and the + * only honest answer available. + */ + static boolean statusForbidsLength(int status) { + return status == 204 || status == 304 || (status >= 100 && status < 200); + } + + private byte[] responseBodyFor(Response response, boolean headOnly) throws IOException { + if(response.fileFd < 0) { + if(headOnly) { + return new byte[0]; + } + if(response.hasDeferredJson) { + // respondJson and jsonValue leave the value unserialised so the HTTP/1.1 + // writer can render it straight into the connection's reusable buffer. + // There is no such buffer here -- the bytes have to become DATA frames -- + // so they are built as their own array. Without this the body is empty, + // and the same handler that works over HTTP/1.1 answers HTTP/2 with + // nothing at all. + return Response.bytes(Json.write(response.deferredJson)); + } + return response.body; + } + try { + if(headOnly) { + return new byte[0]; + } + // Reached only when the caller has no streaming path to offer. The HTTP/2 + // caller does -- see respondFile -- and comes here for HEAD alone, where + // the point of this branch is the close below. + return StaticFiles.readAll(response.fileFd, response.fileOffset, response.fileLength); + } finally { + StaticFiles.closeFile(response.fileFd); + } + } + + /** + * True for the fields whose values this server decides. + * + * A handler that sets Content-Length or Transfer-Encoding through extraHeaders + * gets it serialised AFTER the server's own, so the response carries two + * answers to "where does the body end". A client and a proxy may pick + * different ones, which desynchronises everything after it on that connection + * -- request smuggling, and cache poisoning when the map came from the request. + * Connection is the same: the server decides keep-alive from the request and + * the framing follows from that. + * + * Dropped rather than merged. There is no sensible merge of two lengths, and a + * handler wanting a different body should return a different body. + */ + private static boolean isServerOwnedHeader(String name) { + return name.equalsIgnoreCase("content-length") + || name.equalsIgnoreCase("transfer-encoding") + || name.equalsIgnoreCase("connection") + || name.equalsIgnoreCase("content-type") + || name.equalsIgnoreCase("date"); + } + + /** + * The content type to serialise: the handler's, or the default. + * + * Validated with the same rule as every other header value. Response.respond and + * the public Response constructor both take this from the handler, so it can + * carry request-derived text just as extraHeaders can -- guarding one and not + * the other left the same response-splitting hole open through a different + * argument. A rejected type falls back rather than being dropped, because a + * response without Content-Type is its own problem. + */ + private static String safeContentType(String contentType) { + if(contentType == null) { + return DEFAULT_CONTENT_TYPE; + } + if(isHeaderSafe(contentType)) { + return contentType; + } + System.err.println("replaced a content type containing a control character: " + + sanitizeForLog(contentType)); + return DEFAULT_CONTENT_TYPE; + } + + /** + * True when this text can go into a response head as it stands. + * + * CR and LF end a field; NUL truncates it in every C call underneath. None of + * the three can appear in a header name or value, and a header carrying one is + * either a bug or an injection attempt -- neither is worth serialising. + */ + /** + * True when this is a field NAME as HTTP defines one: a non-empty run of + * tchar (RFC 9110 5.6.2). isHeaderSafe is the right rule for a value and + * the wrong one for a name -- a space, tab or colon passes it and still + * produces a field line no peer reads the way the handler meant. A leading + * space is worse than merely malformed: over HTTP/1 that is obsolete line + * folding, so the name and value are appended to the PREVIOUS header + * instead of forming their own. Over HTTP/2 nghttp2 rejects the name, and + * that can cost the whole response rather than the one header. + */ + private static boolean isHeaderName(String name) { + if(name.length() == 0) { + return false; + } + for(int iter = 0 ; iter < name.length() ; iter++) { + char c = name.charAt(iter); + boolean tchar = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c == '!' || c == '#' || c == '$' || c == '%' || c == '&' + || c == '\'' || c == '*' || c == '+' || c == '-' || c == '.' + || c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; + if(!tchar) { + return false; + } + } + return true; + } + + private static boolean isHeaderSafe(String value) { + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + if(c == '\r' || c == '\n' || c == 0) { + return false; + } + } + return true; + } + + /** The same token rule as isHeaderName, over a slice of the read buffer. */ + private static boolean isRequestHeaderName(byte[] raw, int from, int to) { + if(to <= from) { + return false; + } + for(int iter = from ; iter < to ; iter++) { + int c = raw[iter] & 0xff; + boolean tchar = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || c == '!' || c == '#' || c == '$' || c == '%' || c == '&' + || c == '\'' || c == '*' || c == '+' || c == '-' || c == '.' + || c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; + if(!tchar) { + return false; + } + } + return true; + } + + /** + * Whether this slice holds a byte no field value may carry. HTAB is allowed + * because RFC 9110 permits it inside a value; everything else below 0x20, and + * DEL, is a delimiter to somebody. + */ + private static boolean hasControlByte(byte[] raw, int from, int to) { + for(int iter = from ; iter < to ; iter++) { + int c = raw[iter] & 0xff; + if((c < 0x20 && c != '\t') || c == 0x7f) { + return true; + } + } + return false; + } + + /** The same characters would break the log line they are reported on. */ + private static String sanitizeForLog(String value) { + StringBuilder out = new StringBuilder(value.length()); + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + out.append(c == '\r' || c == '\n' || c < 0x20 ? '?' : c); + } + return out.toString(); + } + + /** + * HTTP/1.1 keeps the connection alive unless asked not to; HTTP/1.0 closes + * unless asked to keep it. Treating a 1.0 client as keep-alive leaves it + * waiting for a close that never comes. + */ + private static boolean wantsKeepAlive(Request request) { + // headerContains rather than getHeader, and this is the hot path. + // + // getHeader materialises the value: asciiString allocates a char[] AND a + // String, and .toLowerCase() allocates a second String -- four objects per + // request to answer a question about a fixed token. The request already + // holds its headers as positions into the read buffer, and + // sliceContainsIgnoreCase answers straight off those bytes, so this is the + // one call site that was throwing that away. Measured at 662 bytes + // allocated per /plaintext request against a 4MB trigger, which is 60-90 + // collections a second, and the collector is what costs the tail. + // + // Same answers as before on both branches: headerContains is false when + // the header is absent, so 1.0 still needs an explicit keep-alive and 1.1 + // still defaults to keeping the connection. + if("HTTP/1.0".equals(request.getVersion())) { + return request.headerContains("connection", "keep-alive"); + } + return !request.headerContains("connection", "close"); + } + + private void writeStatusOnly(Conn conn, int status, String message) { + try { + byte[] body = (message == null ? reason(status) : message) + .getBytes("UTF-8"); + StringBuilder head = new StringBuilder(); + head.append("HTTP/1.1 ").append(status).append(' ').append(reason(status)).append("\r\n"); + head.append("Content-Type: text/plain; charset=utf-8\r\n"); + head.append("Date: ").append(currentHttpDate()).append("\r\n"); + head.append("Content-Length: ").append(body.length).append("\r\n"); + head.append("Connection: close\r\n\r\n"); + conn.write(head.toString().getBytes("UTF-8")); + conn.write(body); + } catch (IOException err) { + // The peer is already gone; there is nowhere to report this. + } + } + + /** + * The methods this server routes. Anything else is 501, not a 404. + * + * Held as constants so a parsed method can BE one of them rather than a fresh + * String per request. + */ + private static final String[] KNOWN_METHODS = { + "GET", "POST", "HEAD", "PUT", "DELETE", "PATCH", "OPTIONS" + }; + + /** + * The same constants as bytes, because comparing against the String walks it a + * character at a time and String.charAt is a call. + * + * Every one of these is matched against raw buffer bytes on the request path, + * and the comparison was reaching into a String for each character: a profile + * of the plaintext benchmark put String.charInternal at 4.92% of in-binary self + * time, third behind syscall dispatch and serveOne itself. A request line costs + * about eleven of those calls -- eight for the version and three for the method + * -- before a single header is looked at. Held as bytes the same comparison is + * a byte load, and the constants are built once at class initialisation. + * + * The IGNORE-CASE constants are stored already folded, so only the data side is + * folded at comparison time rather than both sides on every character. + */ + private static final byte[] HTTP_1_1_BYTES = asciiConstant("HTTP/1.1"); + private static final byte[] HTTP_1_0_BYTES = asciiConstant("HTTP/1.0"); + private static final byte[] CONTENT_LENGTH_BYTES = asciiConstant("content-length"); + private static final byte[] TRANSFER_ENCODING_BYTES = asciiConstant("transfer-encoding"); + private static final byte[] HOST_BYTES = asciiConstant("host"); + private static final byte[][] KNOWN_METHOD_BYTES = asciiConstants(KNOWN_METHODS); + + private static byte[] asciiConstant(String ascii) { + byte[] out = new byte[ascii.length()]; + for(int iter = 0 ; iter < ascii.length() ; iter++) { + out[iter] = (byte)ascii.charAt(iter); + } + return out; + } + + private static byte[][] asciiConstants(String[] values) { + byte[][] out = new byte[values.length][]; + for(int iter = 0 ; iter < values.length ; iter++) { + out[iter] = asciiConstant(values[iter]); + } + return out; + } + + /** + * Reads one request. Null when the peer closed; ProtocolException when what + * arrived is not a request this server will act on. + */ + private Request readRequest(Conn conn, byte[] scratch) throws IOException { + // Cleared before the header block is read and raised once it has been + // parsed, so fill() can tell "start of a request" from "midway through + // one" -- which an empty buffer alone cannot say. + conn.parsedFromBuffer = false; + int headerEnd = indexOfHeaderEnd(conn.buffer, conn.pos); + // An ABSOLUTE bound on the head, not a per-read one. SO_RCVTIMEO restarts + // on every successful read, so a client sending one byte just inside each + // window holds its worker for as long as it likes -- and in pool mode, + // which is what TLS falls back to, the default sixteen such connections + // are the whole server. Armed by the first byte rather than on entry: a + // kept-alive connection may legitimately sit idle between requests, and + // that idleness is the socket timeout's business, not this one. + // + // The head only. A body is bounded by MAX_BODY_BYTES and by the socket + // timeout between reads, and a wall-clock bound on it would refuse a + // large upload over a slow link, which is a real client rather than an + // attack. + long headDeadline = 0; + while(headerEnd < 0) { + if(conn.available() > MAX_HEADER_BYTES) { + throw new ProtocolException(431, "request head too large"); + } + if(conn.available() > 0 && headDeadline == 0) { + headDeadline = System.currentTimeMillis() + SOCKET_TIMEOUT_MILLIS; + } + if(headDeadline != 0 && System.currentTimeMillis() > headDeadline) { + throw new ProtocolException(408, "the request head did not arrive in time"); + } + if(!conn.fill(scratch)) { + return null; + } + headerEnd = indexOfHeaderEnd(conn.buffer, conn.pos); + } + // Parsed IN PLACE, out of the array the kernel filled. Nothing here builds + // a String for a name or a value: the header block used to become a + // String, be split into lines, each line split again and each half + // substring'd, lower-cased and trimmed -- about 2.6KB per request, and the + // largest single source of allocation in this server. Names and tokens are + // ASCII by definition, so a byte comparison with an ASCII fold is exact. + // From here the slices below name positions in THIS array, so fill() must + // preserve it rather than let go of the borrow. + conn.parsedFromBuffer = true; + byte[] raw = conn.buffer; + int blockStart = conn.pos; + int blockEnd = headerEnd; + conn.pos = headerEnd + 4; + + int lineEnd = indexOfCrLfWithin(raw, blockStart, blockEnd); + if(lineEnd < 0) { + lineEnd = blockEnd; // a single request line with no headers + } + if(lineEnd == blockStart) { + throw new ProtocolException(400, "empty request"); + } + + int firstSpace = indexOfByte(raw, blockStart, lineEnd, (byte)' '); + int secondSpace = firstSpace < 0 ? -1 + : indexOfByte(raw, firstSpace + 1, lineEnd, (byte)' '); + if(firstSpace < 0 || secondSpace < 0 + || indexOfByte(raw, secondSpace + 1, lineEnd, (byte)' ') >= 0) { + throw new ProtocolException(400, "malformed request line"); + } + + String version; + int versionStart = secondSpace + 1; + int versionLength = lineEnd - versionStart; + if(sliceEquals(raw, versionStart, versionLength, HTTP_1_1_BYTES)) { + version = "HTTP/1.1"; + } else if(sliceEquals(raw, versionStart, versionLength, HTTP_1_0_BYTES)) { + version = "HTTP/1.0"; + } else { + throw new ProtocolException(505, "unsupported HTTP version"); + } + + String method = knownMethod(raw, blockStart, firstSpace - blockStart); + if(method == null) { + // 501, not 404: the path may well exist, the verb is what is unknown. + throw new ProtocolException(501, "unsupported method"); + } + + int targetStart = firstSpace + 1; + int targetLength = secondSpace - targetStart; + // The origin-form target when it had to be built rather than pointed at. + String synthesized = null; + // The authority of an absolute-form target. RFC 9112 3.2.2 says a server + // receiving one MUST use it and IGNORE the Host field, so keeping it lets + // the two be compared: a proxy sending "GET http://public.example/p" with + // "Host: internal.example" would otherwise leave getHeader("host") saying + // internal.example to whatever routes or authorizes on it. + String absoluteAuthority = null; + // Absolute-form ("GET http://host/path"), which a request through a proxy + // uses and RFC 9112 requires a server to accept. + if(sliceStartsWithIgnoreCase(raw, targetStart, targetLength, "http://") + || sliceStartsWithIgnoreCase(raw, targetStart, targetLength, "https://")) { + int schemeEnd = indexOfByte(raw, targetStart, targetStart + targetLength, (byte)':'); + int authority = schemeEnd + 3; // past "://" + int end = targetStart + targetLength; + int slash = indexOfByte(raw, authority, end, (byte)'/'); + int question = indexOfByte(raw, authority, end, (byte)'?'); + // Whichever comes first ends the authority. Looking only for '/' drops the + // query of "http://host?a=b" on the floor, and reads a '/' INSIDE a query + // value as the start of the path. + int authorityEnd = end; + if(slash >= 0 && (question < 0 || slash < question)) { + authorityEnd = slash; + } else if(question >= 0) { + authorityEnd = question; + } + absoluteAuthority = asciiString(raw, authority, authorityEnd - authority); + if(slash >= 0 && (question < 0 || slash < question)) { + targetLength = end - slash; + targetStart = slash; + } else if(question >= 0) { + // No path but a query. The origin-form is "/" followed by that query, + // which is not a range of this buffer, so it has to be built. + synthesized = "/" + asciiString(raw, question, end - question); + targetStart = -1; + } else { + targetStart = -1; // origin-form is just "/" + } + } + String target; + if(targetStart < 0) { + target = synthesized == null ? "/" : synthesized; + } else { + if(targetLength == 0 + || (raw[targetStart] != '/' + && !(targetLength == 1 && raw[targetStart] == '*' + && "OPTIONS".equals(method)))) { + throw new ProtocolException(400, "malformed request target"); + } + target = conn.internTarget(raw, targetStart, targetLength); + } + // The slice a generated router matches on, so it compares the bytes the + // parser already has instead of the String it just built. Zero when the + // target had to be BUILT rather than pointed at -- absolute-form with no + // path -- because then no range of this buffer holds it and the String is + // the only representation. Passing 0,0 for every request, which is what + // this did, left the byte path unreachable and every route matched as a + // String: correct, and none of the point. + int sliceStart = targetStart < 0 ? 0 : targetStart; + int sliceLength = targetStart < 0 ? 0 : targetLength; + + // Four ints per header, into a buffer the connection reuses. + int[] slices = conn.slices; + int headerCount = 0; + int at = lineEnd + 2; + while(at < blockEnd) { + int end = indexOfCrLfWithin(raw, at, blockEnd); + if(end < 0) { + end = blockEnd; + } + if(end == at) { + at = end + 2; + continue; + } + int first = raw[at] & 0xff; + if(first == ' ' || first == '\t') { + // Obsolete line folding. Two parsers disagreeing about where a + // header ends is how a request is smuggled; RFC 9112 says reject. + throw new ProtocolException(400, "obsolete line folding"); + } + int colon = indexOfByte(raw, at, end, (byte)':'); + if(colon <= at) { + throw new ProtocolException(400, "malformed header"); + } + int nameStart = at; + int nameEnd = colon; + // RFC 9112 5.1: no whitespace between the field name and the colon, and + // a server MUST reject a message that has it. Trimming it instead made + // "Content-Length : 5" a valid Content-Length here while an intermediary + // in front either rejects that line or reads it as a different field. + // Two parsers disagreeing about which headers a request carries is how a + // request is smuggled, which is why the obsolete line folding above is + // refused rather than joined up. + if(nameEnd > nameStart && isSpace(raw[nameEnd - 1])) { + throw new ProtocolException(400, "whitespace before header colon"); + } + int valueStart = colon + 1; + int valueEnd = end; + while(valueStart < valueEnd && isSpace(raw[valueStart])) { + valueStart++; + } + while(valueEnd > valueStart && isSpace(raw[valueEnd - 1])) { + valueEnd--; + } + if(headerCount * 4 + 4 > slices.length) { + int[] grown = new int[slices.length * 2]; + System.arraycopy(slices, 0, grown, 0, slices.length); + slices = grown; + conn.slices = grown; + } + // The name must be a TOKEN and the value must carry no control + // character. Both are smuggling defences, the same one the folding + // and whitespace-before-colon rules above are: this parser finds the + // end of a field by scanning for CRLF, so a bare LF inside a value is + // just a byte to it -- while an intermediary that accepts bare LF as + // a delimiter reads "X: v\nContent-Length: 5" as TWO fields and frames + // the body by that length. One connection, two readings, and the next + // request on it is whatever the attacker put after the body. The + // response side already refuses exactly this shape (isHeaderName); a + // request is the direction that matters more. + if(!isRequestHeaderName(raw, nameStart, nameEnd)) { + throw new ProtocolException(400, "malformed header name"); + } + if(hasControlByte(raw, valueStart, valueEnd)) { + throw new ProtocolException(400, "control character in a header value"); + } + int base = headerCount * 4; + slices[base] = nameStart; + slices[base + 1] = nameEnd - nameStart; + slices[base + 2] = valueStart; + slices[base + 3] = valueEnd - valueStart; + headerCount++; + at = end + 2; + } + + Request request; + if(POOL_REQUEST) { + if(conn.pooledRequest == null) { + conn.pooledRequest = new Request(method, target, version, raw, slices, + headerCount, null, sliceStart, sliceLength); + } else { + conn.pooledRequest.reset(conn, method, target, version, raw, slices, headerCount, + null, sliceStart, sliceLength); + } + request = conn.pooledRequest; + } else { + request = new Request(method, target, version, raw, slices, headerCount, null, + sliceStart, sliceLength); + } + + int contentLengthAt = -1; + boolean chunked = false; + String transferEncoding = null; + int hostCount = 0; + String hostValue = null; + for(int iter = 0 ; iter < headerCount ; iter++) { + int base = iter * 4; + if(sliceEqualsIgnoreCase(raw, slices[base], slices[base + 1], CONTENT_LENGTH_BYTES)) { + // Two different lengths means two readings of where this request + // ends. Refuse rather than pick one. + if(contentLengthAt >= 0 + && !slicesEqual(raw, slices[contentLengthAt + 2], slices[contentLengthAt + 3], + slices[base + 2], slices[base + 3])) { + throw new ProtocolException(400, "conflicting Content-Length"); + } + contentLengthAt = base; + } else if(sliceEqualsIgnoreCase(raw, slices[base], slices[base + 1], + TRANSFER_ENCODING_BYTES)) { + // Every instance, in order, joined with commas. RFC 9110 5.3 makes + // repeated fields mean the same as one field holding the joined + // list, and framing has to be decided on the whole list: assigning + // per field let a second Transfer-Encoding overwrite the first, so + // "chunked" followed by anything else fell back to Content-Length + // here while a proxy in front still framed it as chunked. + String value = asciiString(raw, slices[base + 2], slices[base + 3]); + transferEncoding = transferEncoding == null ? value + : transferEncoding + "," + value; + } else if(sliceEqualsIgnoreCase(raw, slices[base], slices[base + 1], HOST_BYTES)) { + hostCount++; + if(hostValue == null) { + hostValue = asciiString(raw, slices[base + 2], slices[base + 3]); + } + } + } + if(transferEncoding != null) { + // RFC 9112 6.1: chunked MUST be the final coding, and a server that + // cannot decode the rest MUST NOT guess at the framing. An unsupported + // coding, chunked twice, or chunked in the middle all mean this server + // and the next hop could choose different message boundaries. + chunked = requireChunkedIsFinalCoding(transferEncoding); + } + // RFC 9112: an HTTP/1.1 request MUST carry Host, and a server MUST reject + // one that does not. Routing on a name the client never sent is how a + // request reaches the wrong virtual host. + if("HTTP/1.1".equals(version) && hostCount == 0) { + throw new ProtocolException(400, "missing Host header"); + } + // Refused rather than silently preferred one of the two. The authority is + // what this server must act on, but a handler reading getHeader("host") + // would still see the other, and a request that carries two different + // answers to "which host did you mean" has no honest interpretation. + if(absoluteAuthority != null && hostValue != null + && !absoluteAuthority.equalsIgnoreCase(hostValue)) { + throw new ProtocolException(400, + "the request target's authority and the Host header disagree"); + } + // RFC 9112 3.2: more than one Host is a 400. Accepting it lets the two + // request APIs disagree -- getHeader returns the first, getHeaders keeps the + // last -- so a handler and whatever authorized it can read different + // authorities out of the same request. + if(hostCount > 1) { + throw new ProtocolException(400, "duplicate Host header"); + } + + String contentLength = contentLengthAt < 0 ? null : "set"; + int declaredLength = contentLengthAt < 0 ? -1 + : sliceToInt(raw, slices[contentLengthAt + 2], slices[contentLengthAt + 3]); + + if(chunked && contentLength != null) { + // Both framings in one request is precisely how a request is smuggled + // past a proxy that believes one and a server that believes the other. + throw new ProtocolException(400, "both Content-Length and Transfer-Encoding"); + } + + if(request.headerContains("expect", "100-continue")) { + // The client is entitled to wait for this before sending the body. A + // server that stays silent makes every such client pay its whole + // timeout first. + conn.write(CONTINUE_100); + } + + String body = null; + if(chunked) { + byte[] decoded = readChunked(conn, scratch); + if(decoded == null) { + return null; + } + if(decoded.length > 0 && !Utf8.isValid(decoded, 0, decoded.length)) { + throw new ProtocolException(400, "the request body is not valid UTF-8"); + } + body = decoded.length == 0 ? null : new String(decoded, "UTF-8"); + } else if(contentLength != null) { + // sliceToInt returns -1 for anything that is not a plain non-negative + // decimal, which covers the malformed and the negative cases the two + // separate checks here used to make after parsing. + if(declaredLength < 0) { + throw new ProtocolException(400, "malformed Content-Length"); + } + if(declaredLength > MAX_BODY_BYTES) { + throw new ProtocolException(413, "request body too large"); + } + if(!conn.fillTo(declaredLength)) { + return null; + } + if(declaredLength > 0) { + // Checked before it is decoded. new String replaces a malformed + // sequence with U+FFFD rather than failing, so without this the + // handler is handed text the client never sent -- and whatever + // validated it validated the replacement. Note the query-string + // decoder above does the same thing with percent-decoded bytes; + // that one is left alone deliberately, because refusing a query + // parameter is a different policy from refusing a body, and no + // report has been made against it. + if(!Utf8.isValid(conn.buffer, conn.pos, declaredLength)) { + throw new ProtocolException(400, "the request body is not valid UTF-8"); + } + body = new String(conn.buffer, conn.pos, declaredLength, "UTF-8"); + conn.pos += declaredLength; + } + } + // The body is the only field not known when the header block was parsed. + // Both branches below run during PARSING -- before this Request is handed + // to a handler -- so neither one mutates anything a handler can see, and + // "immutable to its handler" is preserved either way. The slices and the + // array are shared, not copied. + if(body == null) { + return request; + } + if(POOL_REQUEST) { + request.reset(conn, method, target, version, raw, slices, headerCount, body, + sliceStart, sliceLength); + return request; + } + return new Request(method, target, version, raw, slices, headerCount, body, + sliceStart, sliceLength); + } + + /** + * Decodes a chunked body: a size in hex, CRLF, that many bytes, CRLF, until a + * zero-length chunk. The trailer section after it is consumed and discarded -- + * ignoring trailers is allowed, but leaving them in the stream would + * desynchronise the next request on a keep-alive connection. + */ + private byte[] readChunked(Conn conn, byte[] scratch) throws IOException { + // Charged against the SAME process-wide budget the fixed-length path uses. + // Bounding only that path left this one open: a chunked body is capped per + // request at MAX_BODY_BYTES and by nothing at all across requests, so + // enough unauthenticated clients sending almost 8 MiB each and pausing + // before the terminating chunk retain gigabytes until their rate deadlines + // expire, with CN1_HTTP_MAX_UPLOAD_MB looking on. + // + // The figure charged is what this read is RETAINING: the chunks already + // accumulated plus what is buffered on the connection for the chunk in + // progress. Charging only the first would miss the second, which grows to + // a whole chunk -- the same "fixed one of the two" that made this comment + // necessary in the first place. + long[] charged = { 0 }; + try { + ByteArrayOutputStream body = new ByteArrayOutputStream(); + // The same floor rate the fixed-length path got, over the WHOLE chunked + // read: the size lines, the data and the trailers. Each of the three fill + // loops below restarts the socket timeout on every successful read, so a + // client sending a one-byte chunk just inside each window could hold a + // worker for years before the 8 MiB cap ever came into view -- and in pool + // mode, which is what TLS uses, enough of those are the server. Bounding + // only the fixed-length path left this one open. + long started = System.currentTimeMillis(); + while(true) { + int lineEnd = indexOfCrLf(conn.buffer, conn.pos); + while(lineEnd < 0) { + // A chunk-size line with no CRLF would otherwise be read for ever: + // fill() reallocates and copies what is already buffered, and none + // of it counts toward MAX_BODY_BYTES because no body byte has been + // framed yet. Metadata gets the same ceiling the header block has. + if(conn.available() > MAX_HEADER_BYTES) { + throw new ProtocolException(400, "chunk size line too long"); + } + requireChunkedProgress(started, body.size() + conn.available()); + reserveUploadUpTo(charged, body.size() + conn.available()); + if(!conn.fill(scratch)) { + return null; + } + lineEnd = indexOfCrLf(conn.buffer, conn.pos); + } + String sizeLine = new String(conn.buffer, conn.pos, lineEnd - conn.pos, "UTF-8"); + // A chunk-size may carry extensions after a ';'; the size is before it. + int semi = sizeLine.indexOf(';'); + if(semi >= 0) { + sizeLine = sizeLine.substring(0, semi); + } + int size; + try { + size = Integer.parseInt(sizeLine.trim(), 16); + } catch (NumberFormatException err) { + throw new ProtocolException(400, "malformed chunk size"); + } + if(size < 0) { + throw new ProtocolException(400, "negative chunk size"); + } + conn.pos = lineEnd + 2; + if(size == 0) { + // Trailers, terminated by a bare CRLF. Bounded in total, not per + // line: an endless run of short well-formed trailers costs exactly + // as much memory as one endless line. + int trailerBytes = 0; + while(true) { + int trailerEnd = indexOfCrLf(conn.buffer, conn.pos); + while(trailerEnd < 0) { + if(conn.available() > MAX_HEADER_BYTES) { + throw new ProtocolException(400, "chunk trailer too long"); + } + requireChunkedProgress(started, body.size() + conn.available()); + reserveUploadUpTo(charged, body.size() + conn.available()); + if(!conn.fill(scratch)) { + // EOF before the blank line that ends the trailers: the + // chunked framing never finished, so this is a truncated + // message, not a complete one. Returning the body here + // ran the handler on it -- and for a mutating request + // that means committing half a message. The fixed-length + // and chunk-data paths both return null; so does this. + return null; + } + trailerEnd = indexOfCrLf(conn.buffer, conn.pos); + } + trailerBytes += (trailerEnd - conn.pos) + 2; + if(trailerBytes > MAX_HEADER_BYTES) { + throw new ProtocolException(400, "chunk trailers too large"); + } + boolean blank = trailerEnd == conn.pos; + conn.pos = trailerEnd + 2; + if(blank) { + return body.toByteArray(); + } + } + } + // Subtraction, not addition: body.size() + size overflows to a negative + // for a chunk size near Integer.MAX_VALUE and sails past the cap, after + // which the loop below grows the buffer toward the declared multi-gigabyte + // chunk. Four bytes and a "7ffffffd" header was enough for an + // unauthenticated client to take the process out. Both sides here are + // non-negative, so there is nothing left to overflow. + if(size > MAX_BODY_BYTES - body.size()) { + throw new ProtocolException(413, "chunked body too large"); + } + // The chunk and its trailing CRLF must both be present before it is taken. + while(conn.available() < size + 2) { + requireChunkedProgress(started, body.size() + conn.available()); + reserveUploadUpTo(charged, body.size() + conn.available()); + if(!conn.fill(scratch)) { + return null; + } + } + // Reserved for the copy BEFORE it is made, like every other growth + // point: a budget checked afterwards has already spent what it meant + // to withhold. + reserveUploadUpTo(charged, body.size() + size + conn.available()); + body.write(conn.buffer, conn.pos, size); + conn.pos += size; + if(conn.buffer[conn.pos] != '\r' || conn.buffer[conn.pos + 1] != '\n') { + throw new ProtocolException(400, "malformed chunk terminator"); + } + conn.pos += 2; + } + } finally { + // Every path out, exactly like the fixed-length reader: the body + // arrived, the peer went away, the deadline passed or the process was + // full. On success the bytes become the request's and stop being an + // upload in flight. + http1UploadBytes.addAndGet(-charged[0]); + } + } + + /** + * Tops a reservation up to what the caller is now holding. + * + * The running total is in the array so that the charge is recorded BEFORE the + * ceiling is tested: if this throws, the caller's finally still releases what + * was just taken. Recording it afterwards leaks the last reservation of every + * refused upload, which is the slowest possible way to run a server out of + * budget. + */ + private static void reserveUploadUpTo(long[] charged, long needed) + throws ProtocolException { + if(needed <= charged[0]) { + return; + } + long delta = needed - charged[0]; + charged[0] = needed; + if(http1UploadBytes.addAndGet(delta) > MAX_HTTP1_UPLOAD_BYTES) { + throw new ProtocolException(503, "too many uploads in flight"); + } + } + + /** + * Refuses a chunked body that is not arriving at the floor rate. + * + * The total is not declared, so the allowance is computed from what has + * ARRIVED: at any moment the elapsed time may be one socket timeout plus what + * those bytes take at MIN_BODY_BYTES_PER_SECOND. A slow but progressing upload + * keeps earning time; one that has stopped delivering does not. + * + * "Arrived" includes what is BUFFERED for the chunk in progress, not just the + * chunks already complete. Counting only completed chunks meant one legal + * large chunk earned no time at all while it streamed: a 1 MiB chunk at four + * times the floor rate was cut off with a 408 after about fifteen seconds, + * because the total stayed zero until the whole of it had landed. + */ + private static void requireChunkedProgress(long started, int received) + throws ProtocolException { + long allowed = SOCKET_TIMEOUT_MILLIS + + (long)received * 1000L / MIN_BODY_BYTES_PER_SECOND; + if(System.currentTimeMillis() - started > allowed) { + throw new ProtocolException(408, "the chunked body did not arrive in time"); + } + } + + private static int indexOfCrLf(byte[] data, int from) { + for(int iter = from ; iter + 1 < data.length ; iter++) { + if(data[iter] == '\r' && data[iter + 1] == '\n') { + return iter; + } + } + return -1; + } + + /** + * The Date header value, formatted at most once a second. + * + * The header has one-second resolution, so formatting it per response is work + * whose result is identical for every request in the same second -- and at + * these rates that is thousands of them. Two threads racing here both compute + * the same string for the same second, so the only cost of the race is a + * duplicated format, never a wrong value. + */ + private static volatile long dateStampSecond = -1; + private static volatile String dateStampValue; + /** + * The same stamp as bytes, so writing it costs a copy rather than a + * per-character conversion. An HTTP date is fixed width and ASCII, which is + * what makes the length a constant. + */ + private static volatile byte[] dateStampBytes = new byte[HTTP_DATE_LENGTH]; + + static String currentHttpDate() { + refreshHttpDate(); + return dateStampValue; + } + + static byte[] currentHttpDateBytes() { + refreshHttpDate(); + return dateStampBytes; + } + + private static void refreshHttpDate() { + long millis = System.currentTimeMillis(); + long second = millis / 1000L; + if(second != dateStampSecond) { + String formatted = Http1Date.format(second * 1000L); + byte[] bytes = new byte[HTTP_DATE_LENGTH]; + // Fixed width by construction; a formatter that ever returned another + // length would otherwise write a short or truncated date silently. + if(formatted.length() != HTTP_DATE_LENGTH) { + throw new IllegalStateException("HTTP date is not " + + HTTP_DATE_LENGTH + " characters: " + formatted); + } + for(int iter = 0 ; iter < HTTP_DATE_LENGTH ; iter++) { + bytes[iter] = (byte)formatted.charAt(iter); + } + dateStampValue = formatted; + dateStampBytes = bytes; + dateStampSecond = second; + } + } + + private void writeResponse(Conn conn, int fd, long session, Response response, + boolean keepAlive, boolean headOnly) throws IOException { + // A deferred JSON body is serialised FIRST: Content-Length has to be + // written before it, and the only honest way to know it is to have the + // bytes. Into a second reusable buffer rather than the head's, because + // the head is not built yet. + byte[] deferred = null; + int deferredLength = 0; + if(response.hasDeferredJson) { + conn.bodySink.reset(); + Json.write(response.deferredJson, conn.bodySink); + deferred = conn.bodySink.bytes(); + deferredLength = conn.bodySink.length(); + } + long bodyLength = response.fileFd >= 0 ? response.fileLength + : (deferred != null ? deferredLength : response.body.length); + // HEAD is not the only thing that suppresses a body; see statusForbidsBody. + boolean noBody = headOnly || statusForbidsBody(response.status); + boolean noLength = statusForbidsLength(response.status); + // A HEAD and a bodiless STATUS are suppressed for different reasons and + // must advertise different lengths. HEAD describes the representation it + // is not sending, so it keeps the real figure. A 205 has no + // representation to describe -- it tells the client to clear its form -- + // so it advertises zero. Reporting the suppressed body's length there + // would leave a keep-alive client waiting for bytes that never come. + if(noBody && !headOnly) { + bodyLength = 0; + } + + // Assembled into the connection's own buffer, as bytes, with no + // intermediate String. See Conn.out: the StringBuilder-to-String-to-bytes + // chain this replaces was the largest single source of allocation in the + // server, and the buffer is reused for the life of the connection. + conn.reset(); + // Pre-encoded, not re-encoded per response. put(String) walks the string + // one charAt at a time; these literals are 82 of the ~97 characters a + // plaintext 200 emits, so writing them that way spent 51 MILLION charAt + // calls a second at this server's throughput to reproduce bytes that never + // change. put(byte[]) is a System.arraycopy. Measured before this: 1.36us + // of user CPU per request against fasthttp's 0.74us, with system time at + // parity -- the gap was all in our own code, and this is the largest + // identifiable piece of it. + if(FAST_HEADERS) { + if(response.status == 200) { + conn.put(H_STATUS_200, 0, H_STATUS_200.length); // overwhelmingly the common case + } else { + conn.put(H_VERSION, 0, H_VERSION.length); + conn.putNumber(response.status); + conn.put(' '); + conn.put(reason(response.status)); + } + conn.put(H_CTYPE, 0, H_CTYPE.length); + // The same default HTTP/2 applies. The public Response constructor lets a + // handler pass null, and reaching putContentType with it threw an NPE that + // dropped the connection without a response -- so one handler behaved two + // ways depending on the protocol it happened to be answering. + conn.putContentType(safeContentType(response.contentType)); + // RFC 9110 6.6.1: an origin server with a clock MUST send Date. + conn.put(H_DATE, 0, H_DATE.length); + conn.put(currentHttpDateBytes(), 0, HTTP_DATE_LENGTH); + // Always an explicit length: without it a keep-alive client waits for + // a close that is not coming. The exception is a status the spec says + // must not carry one, where the absent header IS the framing. + if(!noLength) { + conn.put(H_CLEN, 0, H_CLEN.length); + conn.putNumber(bodyLength); + } + if(keepAlive) { + conn.put(H_KEEPALIVE, 0, H_KEEPALIVE.length); + } else { + conn.put(H_CLOSE, 0, H_CLOSE.length); + } + } else { + // The per-character path this replaces, kept so the two can be + // measured against each other in one binary. + conn.put("HTTP/1.1 "); + conn.putNumber(response.status); + conn.put(' '); + conn.put(reason(response.status)); + conn.put("\r\nContent-Type: "); + // Through the same guard as the fast path above. This branch is the + // measurement copy, and it had BOTH defects that branch was fixed for: a + // null content type reached it as an NPE, and a CR/LF one as a second + // header. A path kept for comparison is still a path that serves. + conn.put(safeContentType(response.contentType)); + conn.put("\r\nDate: "); + conn.put(currentHttpDateBytes(), 0, HTTP_DATE_LENGTH); + if(!noLength) { + conn.put("\r\nContent-Length: "); + conn.putNumber(bodyLength); + } + conn.put(keepAlive ? "\r\nConnection: keep-alive" : "\r\nConnection: close"); + } + if(response.extraHeaders != null) { + java.util.Iterator it = response.extraHeaders.keySet().iterator(); + while(it.hasNext()) { + Object key = it.next(); + Object value = response.extraHeaders.get(key); + if(key != null && value != null) { + String name = String.valueOf(key); + String text = String.valueOf(value); + // A CR or LF here ENDS the field and starts another, so a value + // built from request data -- a decoded query parameter reaches a + // handler with real CRLF in it if the client sent %0d%0a -- lets + // the client write its own headers, or a second response. That is + // response splitting, and it is a cache-poisoning primitive. + // Dropped rather than escaped: there is no correct escaping, and a + // header the handler could not have meant is not worth sending. + if(isServerOwnedHeader(name)) { + System.err.println("dropped a response header the server owns: " + + sanitizeForLog(name)); + } else if(isHeaderName(name) && isHeaderSafe(text)) { + conn.put("\r\n"); + conn.put(name); + conn.put(": "); + conn.put(text); + } else { + System.err.println("dropped a response header whose name is " + + "not a token or whose value carries a control character: " + + sanitizeForLog(name)); + } + } + } + } + if(FAST_HEADERS) { + conn.put(H_END, 0, H_END.length); + } else { + conn.put("\r\n\r\n"); + } + + // Head and body in ONE write when the body is small and already in memory. + // Two writes are two syscalls and, on a fresh connection, two segments: the + // client sees the headers, acknowledges, and only then gets the body. + // Measured against Go, which does one write per response, this was half of + // our remaining syscall count per request. Above the threshold the copy + // would cost more than the syscall it saves, and a file body never enters + // user space at all -- both keep the two-write path. + if(deferred != null) { + if(!noBody && deferredLength > 0) { + conn.put(deferred, 0, deferredLength); + } + writeTo(fd, session, conn.out, 0, conn.outLength); + return; + } + if(response.fileFd < 0 && !noBody + && response.body.length > 0 + && response.body.length <= COMBINED_WRITE_LIMIT) { + conn.put(response.body, 0, response.body.length); + writeTo(fd, session, conn.out, 0, conn.outLength); + return; + } + writeTo(fd, session, conn.out, 0, conn.outLength); + + if(response.fileFd >= 0) { + try { + if(!noBody) { + StaticFiles.sendBody(fd, session, response.fileFd, response.fileOffset, response.fileLength); + } + } finally { + // The server owns the descriptor once a handler hands it over, so + // this is the only place it is closed -- including when the send + // failed halfway. + StaticFiles.closeFile(response.fileFd); + } + return; + } + if(!noBody && response.body.length > 0) { + writeTo(fd, session, response.body, 0, response.body.length); + } + } + + /** Pre-encoded: this goes out on the body path of every expecting client. */ + private static final byte[] CONTINUE_100 = asciiBytes("HTTP/1.1 100 Continue\r\n\r\n"); + + /** + * The response head's fixed bytes, encoded once at class init instead of + * character by character per response. CN1_HTTP_FAST_HEADERS=0 restores the + * per-character path, which is what the measurement compares against. + */ + private static final boolean FAST_HEADERS = envInt("CN1_HTTP_FAST_HEADERS", 1) != 0; + private static final byte[] H_STATUS_200 = asciiBytes("HTTP/1.1 200 OK"); + private static final byte[] H_VERSION = asciiBytes("HTTP/1.1 "); + private static final byte[] H_CTYPE = asciiBytes("\r\nContent-Type: "); + private static final byte[] H_DATE = asciiBytes("\r\nDate: "); + private static final byte[] H_CLEN = asciiBytes("\r\nContent-Length: "); + private static final byte[] H_KEEPALIVE = asciiBytes("\r\nConnection: keep-alive"); + private static final byte[] H_CLOSE = asciiBytes("\r\nConnection: close"); + private static final byte[] H_END = asciiBytes("\r\n\r\n"); + + private static byte[] asciiBytes(String value) { + byte[] out = new byte[value.length()]; + for(int iter = 0 ; iter < out.length ; iter++) { + out[iter] = (byte)value.charAt(iter); + } + return out; + } + + /** + * True when the joined Transfer-Encoding list ends in `chunked` and carries + * nothing this server cannot decode. + * + * Throws rather than returning false for a list it will not act on: silently + * ignoring a coding leaves the body to be read as the next request on the + * connection, which is the smuggling case this exists to close. + */ + private static boolean requireChunkedIsFinalCoding(String value) + throws ProtocolException { + String[] codings = splitOn(value, ','); + int seen = 0; + for(int iter = 0 ; iter < codings.length ; iter++) { + String coding = codings[iter].trim(); + // A transfer coding may carry parameters after a semicolon; the coding + // itself is what decides the framing. + int semi = coding.indexOf(';'); + if(semi >= 0) { + coding = coding.substring(0, semi).trim(); + } + if(coding.length() == 0) { + continue; + } + if(!"chunked".equalsIgnoreCase(coding)) { + throw new ProtocolException(501, "unsupported transfer coding"); + } + if(iter != codings.length - 1) { + throw new ProtocolException(400, "chunked is not the final transfer coding"); + } + seen++; + } + if(seen == 0) { + throw new ProtocolException(400, "empty Transfer-Encoding"); + } + return true; + } + + private static boolean isSpace(byte b) { + return b == ' ' || b == '\t'; + } + + static int indexOfByte(byte[] data, int from, int to, byte wanted) { + for(int iter = from ; iter < to ; iter++) { + if(data[iter] == wanted) { + return iter; + } + } + return -1; + } + + static int indexOfCrLfWithin(byte[] data, int from, int to) { + for(int iter = from ; iter + 1 < to ; iter++) { + if(data[iter] == '\r' && data[iter + 1] == '\n') { + return iter; + } + } + return -1; + } + + static boolean sliceStartsWithIgnoreCase(byte[] data, int start, int length, String ascii) { + return length >= ascii.length() + && sliceEqualsIgnoreCase(data, start, ascii.length(), ascii); + } + + static boolean slicesEqual(byte[] data, int aStart, int aLength, int bStart, int bLength) { + if(aLength != bLength) { + return false; + } + for(int iter = 0 ; iter < aLength ; iter++) { + if(data[aStart + iter] != data[bStart + iter]) { + return false; + } + } + return true; + } + + // ---- byte-slice helpers ------------------------------------------------- + // + // Header names and the tokens compared against them are ASCII by definition + // (RFC 9110 field-name is a token), so a byte-wise comparison with an ASCII + // fold is exact -- no locale, no decoding, no allocation. These are what let + // the request path answer "is this connection keep-alive" without building a + // String. + + private static int foldAscii(int c) { + return c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; + } + + /** + * The case-folded bytes of an ASCII string, or null if it is not ASCII. + * + * Cached per String IDENTITY, because the callers pass literals: "connection" + * at a given call site is the same object every time, so the fold happens once + * for the life of the process rather than once per header per request. A miss + * simply folds again -- the cache is a hint, never a correctness dependency, + * which is what lets it stay lock-free. + */ + /** + * Case-folded header names, packed END TO END in ONE byte[]. + * + * Flat on purpose. An array of arrays scatters every entry across the heap and + * costs a pointer chase per lookup; this holds all of them contiguously, so a + * comparison walks memory the prefetcher already has. It is also one object for + * the collector to mark instead of seventeen. + * + * Keyed by String IDENTITY, because the callers pass literals -- "connection" at + * a given call site is the same object every time, so the fold happens once for + * the life of the process rather than once per header per request. A miss simply + * folds again: the cache is a hint, never a correctness dependency, which is + * what lets it stay lock-free. + */ + private static final int FOLD_CACHE_SLOTS = 16; + + /** + * One cached fold. Both fields are final, which is the whole point. + * + * The cache used to be three parallel static arrays and a rotating index, and + * a lookup returned the SLOT it had matched. The caller then compared against + * that slot while walking the request's headers -- a window in which another + * worker could retire the slot and write a different name into its bytes. Two + * names of equal length are then indistinguishable, so getHeader answered + * with the wrong field, or reported a header that was sent as absent, and + * nothing threw. Clearing the key first does not help a reader that already + * holds the index. + * + * Handing back an immutable entry closes that by construction: what the + * caller compares against cannot be rewritten, because nothing ever writes to + * a published entry. + */ + private static final class Folded { + final String key; + final byte[] bytes; + + Folded(String key, byte[] bytes) { + this.key = key; + this.bytes = bytes; + } + } + + /** + * Published by replacement, never by mutation, so a reader either sees an + * entry complete or does not see it at all. Two threads that fold the same + * name at once may lose one of the two writes; that costs a later refold and + * nothing else, which is what keeps this lock free. + */ + private static volatile Folded[] foldCache = new Folded[0]; + + /** + * The folded bytes of `ascii`, or null when it cannot be cached -- not ASCII, + * or the cache is full. Null means the caller takes the general path, which + * is only slower. + */ + static byte[] foldedBytes(String ascii) { + Folded[] snapshot = foldCache; + for(int iter = 0 ; iter < snapshot.length ; iter++) { + // Identity, not equals: a given call site hands over the same constant + // every time, so this is a pointer compare and the fold happens once + // for the life of the process. + if(snapshot[iter].key == ascii) { + return snapshot[iter].bytes; + } + } + if(snapshot.length >= FOLD_CACHE_SLOTS) { + return null; + } + int length = ascii.length(); + for(int iter = 0 ; iter < length ; iter++) { + if(ascii.charAt(iter) > 127) { + return null; + } + } + byte[] bytes = new byte[length]; + for(int iter = 0 ; iter < length ; iter++) { + bytes[iter] = (byte) foldAscii(ascii.charAt(iter)); + } + Folded[] grown = new Folded[snapshot.length + 1]; + System.arraycopy(snapshot, 0, grown, 0, snapshot.length); + grown[snapshot.length] = new Folded(ascii, bytes); + foldCache = grown; + return bytes; + } + + /** Case-insensitive compare of a slice against already-folded needle bytes. */ + static boolean sliceEqualsFolded(byte[] data, int start, int length, byte[] needle) { + if(length != needle.length) { + return false; + } + for(int iter = 0 ; iter < length ; iter++) { + if(foldAscii(data[start + iter] & 0xff) != needle[iter]) { + return false; + } + } + return true; + } + + /** + * Folded compare against a constant that is ALREADY folded, so only the bytes + * that arrived off the socket have to be folded here. + */ + static boolean sliceEqualsIgnoreCase(byte[] data, int start, int length, byte[] asciiLower) { + if(length != asciiLower.length) { + return false; + } + for(int iter = 0 ; iter < length ; iter++) { + if(foldAscii(data[start + iter] & 0xff) != (asciiLower[iter] & 0xff)) { + return false; + } + } + return true; + } + + static boolean sliceEqualsIgnoreCase(byte[] data, int start, int length, String ascii) { + if(length != ascii.length()) { + return false; + } + for(int iter = 0 ; iter < length ; iter++) { + if(foldAscii(data[start + iter] & 0xff) != foldAscii(ascii.charAt(iter))) { + return false; + } + } + return true; + } + + static boolean sliceContainsIgnoreCase(byte[] data, int start, int length, String ascii) { + int needle = ascii.length(); + if(needle == 0 || needle > length) { + return needle == 0; + } + int last = start + length - needle; + for(int at = start ; at <= last ; at++) { + int iter = 0; + while(iter < needle + && foldAscii(data[at + iter] & 0xff) == foldAscii(ascii.charAt(iter))) { + iter++; + } + if(iter == needle) { + return true; + } + } + return false; + } + + /** + * A non-negative decimal from a slice, or -1 when it is not one. + * + * Integer.parseInt would need a String first, which is the allocation this + * whole representation exists to avoid -- and it is on the path of every + * request that carries a body. + */ + static int sliceToInt(byte[] data, int start, int length) { + if(length <= 0 || length > 10) { + return -1; + } + long value = 0; + for(int iter = 0 ; iter < length ; iter++) { + int c = data[start + iter] & 0xff; + if(c < '0' || c > '9') { + return -1; + } + value = value * 10 + (c - '0'); + if(value > Integer.MAX_VALUE) { + return -1; + } + } + return (int)value; + } + + /** + * The request target as a String, memoised PER CONNECTION. + * + * A connection asks for the same handful of targets over and over, so this is a + * hit almost every time and the steady state allocates nothing. A miss does + * exactly what the code did before and is only slower, never wrong. + * + * Worth doing because the target was the last per-request String on the + * plaintext path, and it cost three objects rather than one: asciiString builds + * a char[] and String's public constructor copies it into a second. + * + * PER CONNECTION rather than one shared static table, and that is a + * correctness requirement rather than a preference. `java.lang.String.value` is + * NOT final in this runtime (only offset and count are), so a String published + * through an unsynchronised static array can be observed by another worker with + * a null value -- on arm64 that is a real reordering, not a theoretical one. A + * Conn reaches its next worker through the executor, which gives the + * happens-before edge this needs for free. + * + * Bounded, because targets are attacker controlled: a query string or path + * parameter makes every request unique. Past the cap it stops inserting and + * every miss allocates as before -- a performance cliff, never a memory one. + */ + // DEFAULT OFF. Measured against the same binary at 16 connections, the cache + // is worth +24% when the zero-copy read is on (159,291 vs 128,111) and -6% + // when it is off (172,681 vs 183,406). Since the zero-copy read is itself off + // by default, the case that applies is the one where this costs throughput. + // Kept behind a switch rather than deleted because the allocation it removes is + // real -- 2 char[] and a String per request -- and a cheaper lookup might yet + // win; what is NOT supported is turning it on without re-measuring. + /** + * On by default. Sixty-four slots per connection, one reference each. + * + * With this at 0 internTarget takes its disabled path and calls asciiString + * for EVERY request, which allocates a char[], a String and the String's own + * storage. A per-class allocation profile of /plaintext at 64 connections put + * char[] + String + byte[] at 57% of all bytes allocated -- 424MB, 123MB and + * 302MB against a 1.49GB total -- and the request target is the only thing + * left materialising on that path once the keep-alive check stopped doing it. + * + * A benchmark client sends a handful of distinct targets, and a real service + * has a bounded route set, so the slot array is small and the hit rate is + * high. 64 references per connection is 512 bytes, against the ~180 bytes per + * REQUEST the miss path was costing. + * + * MEASURED: paired A/B, arms alternated inside each rep, six readings at 64 + * and 256 connections -- +7% to +13% throughput, 6 of 6 in favour, p99 better + * in 5 of 6, and allocation 639 -> 411 bytes per request with String + * allocations falling from 2,575,425 to 542. The backend suite passed twice. + * + * NOT MEASURED: a workload whose targets are all DISTINCT, which is what + * query strings produce and what an attacker can force. Five attempts to + * measure it failed for harness reasons rather than server ones -- 404s reply + * Connection: close so varied targets tore down the connection, and driving + * wrk from Lua produced 1.5M write errors against 20k requests. The arm is + * still worth building if this ever looks suspect. + * + * What the MISS path costs, from the code rather than a measurement: a hash + * over the target, a length compare that fails immediately, then exactly the + * asciiString the disabled path performs, plus a reference store. So a miss + * adds one pass over a short byte range and allocates nothing extra -- it + * cannot allocate MORE than the cache being off, because it stores the very + * String that path would have created. That bounds the worst case to a small + * constant, which is why this ships on rather than off. + * + * CN1_HTTP_TARGET_CACHE=0 restores the old behaviour for A/B. + */ + private static final int TARGET_CACHE_SLOTS = + envIntAtLeast("CN1_HTTP_TARGET_CACHE", 64, 0); + + /** + * Read straight into the thread's reusable buffer instead of a fresh array. + * A switch because it is the kind of change that has to be A/B measurable + * against the allocation it removes -- an optimisation that costs more than it + * saves looks exactly like one that works until somebody measures the thing it + * was supposed to improve. + */ + /** + * Read straight into the thread's reusable buffer instead of a fresh array. + * + * ON by default. It removes 95% of the per-request byte[] allocations + * (1.04 to 0.054 per request). + * + * MODES, because the throughput answer turned out to depend on the route and + * two earlier readings here were both wrong: + * + * 0 read with recv() into the worker's reusable scratch, then copy into a + * fresh byte[] sized to the read. Allocates once per request. + * 1 read straight into this thread's foreign (off-heap) buffer and parse + * where the bytes land. Allocates nothing. + * 2 DIAGNOSTIC. Identical native to mode 1, identical Java to mode 0: read + * into the foreign buffer and immediately copy it into a heap array. It + * exists to split mode 1's two differences from mode 0 -- the syscall and + * the off-heap object living in a Java field -- because measuring only 0 + * against 1 cannot say which of them moved the number. + * + * Paired measurement on an idle Linux box, same binary, 3 reps, median req/s: + * + * /plaintext mode 0 = 262961 mode 1 = 233800 mode 1 is 11% SLOWER + * /json mode 0 = 167851 mode 1 = 176876 mode 1 is 5% FASTER + * + * That split is the whole reason the modes are here. Earlier comments in this + * spot claimed first a flat 30% loss and then no cost at all; the first was + * measured against a machine running a compile, the second was an A/B too + * noisy to resolve an 11% effect and should have been reported as a failed + * measurement rather than a result. + * + * WHAT THE BISECTION FOUND. Mode 2 lands on mode 0 on BOTH routes -- 250962 + * against 249524 on /plaintext, 159168 against 158972 on /json -- so the + * native read costs nothing and is not what moved either number. Since mode 2 + * differs from mode 1 only in copying the bytes into a heap array, the whole + * effect, in both directions, is the cost of keeping a FOREIGN off-heap array + * in a Java field: + * + * /json mode 1 gains 5.3% over mode 2 -- the route is allocation + * bound, so not allocating a per-request array is worth more + * than the collector's extra work. + * /plaintext mode 1 loses 4.2% to mode 2 -- little GC pressure here, so + * the saved allocation buys little while the off-heap cost is + * paid on every traversal: `Conn.buffer` points outside the + * heap, so the fast range check in the mark path fails and the + * object has to be resolved as an immortal root instead. + * + * The default is therefore a judgement about the workload rather than a fact + * about the code, which is why the switch is left in place. + */ + private static final int ZERO_COPY_MODE = envInt("CN1_HTTP_ZERO_COPY", 1); + /** + * On under virtual threads too, since the reason it was not is gone. + * + * WHAT THIS USED TO SAY, AND WHY IT WAS WRONG. It read that combining the two + * was unsafe because the zero-copy read hands back a HOST thread's buffer and + * a virtual thread could resume elsewhere, and it cited an attempt that + * "passed the virtual-thread suite 21/21 and then produced a TRUNCATED + * RESPONSE on the dispatching path: authGuardsMutatingRoutes read a reply it + * could not parse, once". + * + * That truncation was not the buffer refactor. It was the missing + * parsedFromBuffer guard in fill() -- see the comment there, which records + * the same two symptoms (transactionRollsBack timing out at 15.05s with + * status -1, authGuardsMutatingRoutes reading an empty body, about 2 runs in + * 6) and says in as many words that it "never appeared under virtual threads + * because ZERO_COPY_READ is off there". Turning zero-copy on under virtual + * threads is exactly what first exposed that bug; the refactor was blamed, + * reverted, and the real defect found and fixed afterwards without anyone + * going back to correct the verdict. + * + * The sharing hazard is handled by that same guard rather than by keeping the + * paths apart. Several virtual threads do multiplex onto one host thread, but + * fill() either releases the borrow when nothing has been parsed out of it or + * calls detachPreservingOffsets before any second read, so a virtual thread + * that parks mid-request already owns a private copy and no other thread's + * read can overwrite live slices. + * + * What it costs to leave off: a per-request byte[] copy on every request. The + * census measured 223 bytes per request when this path was copying and none + * when it was not, against 662 bytes per request total on /plaintext. + * + * CN1_HTTP_ZERO_COPY=0 still disables it entirely. + */ + private static final boolean ZERO_COPY_READ = ZERO_COPY_MODE != 0; + + /** + * Reuse one Request per connection instead of allocating one per request. + * + * Request and Response were the whole of what /plaintext still allocated once + * the borrowed-buffer copy went -- 80 and 88 bytes, one of each, every + * request -- so this is half of what was left. The allocation half is exact + * and was measured directly: Request disappears from the profile and the + * route falls from 168.2 to 88.2 bytes per request. Throughput, thirteen + * interleaved pairs in one binary with the arm order rotating, is a median + * +13.6% and ahead in 12 of 13, p99 better in 10. + * + * A profiled build shows only +2.3% for the same change, and that is not a + * contradiction: the profiler taxes every allocation, so the server is slower, + * allocates less per second, and the collector it is being spared matters + * less. The non-profiled figure is the one that describes a real deployment. + * + * CN1_HTTP_POOL_REQUEST=0 restores the allocating path -- kept for the same + * reason ZERO_COPY_MODE keeps its switch, so the comparison stays runnable + * rather than having to be rebuilt. + */ + private static final boolean POOL_REQUEST = envInt("CN1_HTTP_POOL_REQUEST", 1) != 0; + + static String asciiString(byte[] data, int start, int length) { + char[] chars = new char[length]; + for(int iter = 0 ; iter < length ; iter++) { + chars[iter] = (char)(data[start + iter] & 0xff); + } + return new String(chars); + } + + static String lowerCaseString(byte[] data, int start, int length) { + char[] chars = new char[length]; + for(int iter = 0 ; iter < length ; iter++) { + chars[iter] = (char)foldAscii(data[start + iter] & 0xff); + } + return new String(chars); + } + + /** + * The interned constant for a known method, or null. + * + * Returning a constant rather than a fresh String means the common methods + * cost nothing, and it makes the identity comparisons elsewhere in this file + * safe as well as the equals ones. + */ + static String knownMethod(byte[] data, int start, int length) { + // The folded compare that used to guard this one was redundant: an EXACT + // match implies a folded match, so it could only ever agree with the test + // below it, at the cost of a second walk of the same bytes -- with a + // foldAscii call per character on both sides -- for every request. + for(int iter = 0 ; iter < KNOWN_METHOD_BYTES.length ; iter++) { + if(sliceEquals(data, start, length, KNOWN_METHOD_BYTES[iter])) { + return KNOWN_METHODS[iter]; + } + } + return null; + } + + /** Exact, not folded: HTTP methods are case SENSITIVE. */ + /** Exact compare against a constant already held as bytes. */ + private static boolean sliceEquals(byte[] data, int start, int length, byte[] ascii) { + if(length != ascii.length) { + return false; + } + for(int iter = 0 ; iter < length ; iter++) { + if(data[start + iter] != ascii[iter]) { + return false; + } + } + return true; + } + + private static boolean sliceEquals(byte[] data, int start, int length, String ascii) { + if(length != ascii.length()) { + return false; + } + for(int iter = 0 ; iter < length ; iter++) { + if((data[start + iter] & 0xff) != ascii.charAt(iter)) { + return false; + } + } + return true; + } + + private static String reason(int status) { + switch(status) { + case 200: return "OK"; + case 201: return "Created"; + case 204: return "No Content"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 409: return "Conflict"; + case 413: return "Payload Too Large"; + case 500: return "Internal Server Error"; + case 503: return "Service Unavailable"; + default: return status < 400 ? "OK" : "Error"; + } + } + + private static int indexOfHeaderEnd(byte[] data, int from) { + for(int iter = from ; iter + 3 < data.length ; iter++) { + if(data[iter] == '\r' && data[iter + 1] == '\n' + && data[iter + 2] == '\r' && data[iter + 3] == '\n') { + return iter; + } + } + return -1; + } + + private static String[] splitLines(String value) { + List parts = new ArrayList(); + int pos = 0; + while(true) { + int next = value.indexOf("\r\n", pos); + if(next < 0) { + if(pos < value.length()) { + parts.add(value.substring(pos)); + } + break; + } + parts.add(value.substring(pos, next)); + pos = next + 2; + } + String[] out = new String[parts.size()]; + for(int iter = 0 ; iter < out.length ; iter++) { + out[iter] = (String)parts.get(iter); + } + return out; + } + + private static String[] splitOn(String value, char sep) { + List parts = new ArrayList(); + int pos = 0; + while(true) { + int next = value.indexOf(sep, pos); + if(next < 0) { + parts.add(value.substring(pos)); + break; + } + parts.add(value.substring(pos, next)); + pos = next + 1; + } + String[] out = new String[parts.size()]; + for(int iter = 0 ; iter < out.length ; iter++) { + out[iter] = (String)parts.get(iter); + } + return out; + } +} diff --git a/vm/backend/src/com/codename1/backend/Json.java b/vm/backend/src/com/codename1/backend/Json.java new file mode 100644 index 00000000000..849b393ca5f --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Json.java @@ -0,0 +1,717 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A self-contained JSON reader/writer for server-side binaries. + * + * Deliberately not com.codename1.io.JSONParser: that class reaches + * com.codename1.processing.Result, which reaches com.codename1.xml.Element, so + * reusing it would link an XML DOM into every server binary to parse a request + * body. The mapping package has the same problem (Mapper imports Element). + * + * Values map as: object to LinkedHashMap (insertion-ordered so a round trip is + * stable), array to ArrayList, string to String, number to Double or Long, + * true/false to Boolean, null to null. + */ +public final class Json { + private final String src; + private int pos; + + private Json(String src) { + this.src = src; + } + + /** Parses a JSON object. Throws IOException on anything malformed. */ + public static Map parseObject(String json) throws IOException { + Object value = parse(json); + if(!(value instanceof Map)) { + throw new IOException("Expected a JSON object"); + } + return (Map)value; + } + + public static Object parse(String json) throws IOException { + if(json == null) { + throw new IOException("No JSON to parse"); + } + Json p = new Json(json); + p.skipWhitespace(); + Object value = p.readValue(); + p.skipWhitespace(); + if(p.pos < p.src.length()) { + throw new IOException("Trailing content at offset " + p.pos); + } + return value; + } + + /** + * How deep a document may nest before it is refused. + * + * The parser is recursive, so nesting depth is stack depth, and a body is + * whatever the client sent. Without a bound a few kilobytes of "[[[[..." reach + * StackOverflowError -- which is an Error, so neither the handler's catch nor + * the server's catch of Exception sees it, and the thread dies rather than the + * request failing. 512 is far past any real document and far short of the + * stack. + * + * "Far short of the stack" holds on the PACKAGED runtime too, where handlers + * run on a 64KB virtual-thread stack rather than a platform thread's 16MB, + * and it is worth saying why because the arithmetic looks alarming until you + * know: a translated Java frame keeps its locals and operand stack in + * threadStateData->threadObjectStack, a HEAP array, so nesting costs a small + * C frame rather than a whole Java one -- that is the reason these stacks + * can be small at all (see cn1_virtual_thread.h). Both of the limits that + * deep recursion can actually reach are guarded and throw a catchable + * StackOverflowError instead of running off the end: the call depth + * (CN1_MAX_STACK_CALL_DEPTH) and the object stack itself. + * + * Measured rather than argued: a body nested 511 deep -- one under this cap, + * so the cap does not hide the recursion -- is answered cleanly by the + * packaged server on the default 64KB stack, with the server still serving + * afterwards. BackendHttpIntegrationTest keeps that as a regression test. + */ + private static final int MAX_DEPTH = 512; + + private int depth; + + private Object readValue() throws IOException { + if(pos >= src.length()) { + throw new IOException("Unexpected end of JSON"); + } + char c = src.charAt(pos); + switch(c) { + case '{': return readNested(true); + case '[': return readNested(false); + case '"': return readString(); + case 't': return readLiteral("true", Boolean.TRUE); + case 'f': return readLiteral("false", Boolean.FALSE); + case 'n': return readLiteral("null", null); + default: return readNumber(); + } + } + + /** Depth is counted here so both containers share one bound and one release. */ + private Object readNested(boolean object) throws IOException { + if(depth >= MAX_DEPTH) { + throw new IOException("JSON nested deeper than " + MAX_DEPTH); + } + depth++; + try { + return object ? (Object)readObject() : (Object)readArray(); + } finally { + depth--; + } + } + + private Map readObject() throws IOException { + Map out = new LinkedHashMap(); + pos++; // { + skipWhitespace(); + if(peek() == '}') { + pos++; + return out; + } + while(true) { + skipWhitespace(); + if(peek() != '"') { + throw new IOException("Expected a key at offset " + pos); + } + String key = readString(); + skipWhitespace(); + if(peek() != ':') { + throw new IOException("Expected ':' at offset " + pos); + } + pos++; + skipWhitespace(); + out.put(key, readValue()); + skipWhitespace(); + char c = peek(); + pos++; + if(c == '}') { + return out; + } + if(c != ',') { + throw new IOException("Expected ',' or '}' at offset " + (pos - 1)); + } + } + } + + private List readArray() throws IOException { + List out = new ArrayList(); + pos++; // [ + skipWhitespace(); + if(peek() == ']') { + pos++; + return out; + } + while(true) { + skipWhitespace(); + out.add(readValue()); + skipWhitespace(); + char c = peek(); + pos++; + if(c == ']') { + return out; + } + if(c != ',') { + throw new IOException("Expected ',' or ']' at offset " + (pos - 1)); + } + } + } + + private String readString() throws IOException { + pos++; // opening quote + StringBuilder out = new StringBuilder(); + while(true) { + if(pos >= src.length()) { + throw new IOException("Unterminated string"); + } + char c = src.charAt(pos++); + if(c == '"') { + return out.toString(); + } + if(c != '\\') { + // RFC 8259: everything below U+0020 has to arrive escaped. Taking + // it literally accepted documents that a conforming parser -- or + // whatever validates upstream of this one -- rejects, which is + // how the two disagree about where a string ends. + if(c < 0x20) { + throw new IOException("A control character must be escaped in a " + + "JSON string, at offset " + (pos - 1)); + } + out.append(c); + continue; + } + if(pos >= src.length()) { + throw new IOException("Unterminated escape"); + } + char esc = src.charAt(pos++); + switch(esc) { + case '"': out.append('"'); break; + case '\\': out.append('\\'); break; + case '/': out.append('/'); break; + case 'b': out.append('\b'); break; + case 'f': out.append('\f'); break; + case 'n': out.append('\n'); break; + case 'r': out.append('\r'); break; + case 't': out.append('\t'); break; + case 'u': + if(pos + 4 > src.length()) { + throw new IOException("Truncated \\u escape"); + } + try { + out.append((char)Integer.parseInt(src.substring(pos, pos + 4), 16)); + } catch (NumberFormatException err) { + throw new IOException("Malformed \\u escape at offset " + pos); + } + pos += 4; + break; + default: + throw new IOException("Unknown escape \\" + esc); + } + } + } + + private Object readLiteral(String literal, Object value) throws IOException { + if(!src.startsWith(literal, pos)) { + throw new IOException("Expected " + literal + " at offset " + pos); + } + pos += literal.length(); + return value; + } + + /** + * A JSON number, by the grammar rather than by what Java happens to parse. + * + * Scanning a run of "-+0-9.eE" and handing it to Long.parseLong accepted "+1", + * "01", ".5" and "1." -- none of which is a JSON number, and all of which a + * conforming client or an upstream validator rejects. A parser that takes + * documents its own clients cannot produce is worse than a strict one. + * + * RFC 8259: [ '-' ] ( '0' | [1-9][0-9]* ) [ '.' [0-9]+ ] [ ('e'|'E') [+-] [0-9]+ ] + */ + private static boolean isDigit(char c) { + return c >= '0' && c <= '9'; + } + + private Object readNumber() throws IOException { + int start = pos; + boolean floating = false; + if(pos < src.length() && src.charAt(pos) == '-') { + pos++; // '+' is not a JSON sign + } + int intStart = pos; + if(pos < src.length() && src.charAt(pos) == '0') { + pos++; + if(pos < src.length() && isDigit(src.charAt(pos))) { + throw new IOException("A leading zero is not a JSON number, at offset " + + start); + } + } else { + while(pos < src.length() && isDigit(src.charAt(pos))) { + pos++; + } + } + if(pos == intStart) { + throw new IOException("Expected a digit at offset " + intStart); + } + if(pos < src.length() && src.charAt(pos) == '.') { + floating = true; + pos++; + int fracStart = pos; + while(pos < src.length() && isDigit(src.charAt(pos))) { + pos++; + } + if(pos == fracStart) { + throw new IOException("Expected a digit after '.' at offset " + fracStart); + } + } + if(pos < src.length() && (src.charAt(pos) == 'e' || src.charAt(pos) == 'E')) { + floating = true; + pos++; + if(pos < src.length() && (src.charAt(pos) == '-' || src.charAt(pos) == '+')) { + pos++; + } + int expStart = pos; + while(pos < src.length() && isDigit(src.charAt(pos))) { + pos++; + } + if(pos == expStart) { + throw new IOException("Expected a digit in the exponent at offset " + + expStart); + } + } + String text = src.substring(start, pos); + try { + // Integers stay integers: a long round-tripped through double loses + // precision above 2^53, and ids are exactly the values that get large. + if(floating) { + double parsed = Double.parseDouble(text); + // parseDouble answers Infinity for "1e999" rather than throwing, so a + // number JSON cannot represent was accepted and handed on as one an + // amount or a threshold could be built from -- and the writer emits + // null for a non-finite double, so parsing and writing it back + // silently turned the value into null. + if(Double.isNaN(parsed) || Double.isInfinite(parsed)) { + throw new IOException("Number out of range for JSON: '" + text + "'"); + } + return (Object)Double.valueOf(parsed); + } + return (Object)Long.valueOf(Long.parseLong(text)); + } catch (NumberFormatException err) { + throw new IOException("Malformed number '" + text + "'"); + } + } + + private char peek() throws IOException { + if(pos >= src.length()) { + throw new IOException("Unexpected end of JSON"); + } + return src.charAt(pos); + } + + private void skipWhitespace() { + while(pos < src.length()) { + char c = src.charAt(pos); + if(c == ' ' || c == '\t' || c == '\n' || c == '\r') { + pos++; + } else { + break; + } + } + } + + // ------------------------------------------------------------------ + // Writing + // ------------------------------------------------------------------ + + public static String write(Object value) { + StringBuilder out = new StringBuilder(); + writeValue(out, value); + return out.toString(); + } + + /** + * Writes JSON as UTF-8 bytes straight into a reusable buffer. + * + * The String-returning form above builds a StringBuilder, grows its char[] + * several times, copies it into a String and then encodes that to bytes -- + * four allocations for a document the server is about to write to a socket + * and discard. Measured on a small JSON response, that chain was most of the + * per-request allocation once the response head had been dealt with, and it + * kept the collector's run-ahead cap firing. + * + * Same output as {@link #write(Object)}, byte for byte. + */ + public static void write(Object value, ByteSink out) { + writeValue(out, value); + } + + /** + * A value that knows how to write itself as JSON. + * + * The point of this interface is to let generated code skip the Map. A codec + * emitted by the annotation processor knows its field names and types at build + * time, so it can write straight into the sink; without somewhere to hang that, + * a handler's return value has to become a LinkedHashMap first and be walked + * back with instanceof dispatch per value. Measured on the benchmark's + * one-field object, just removing the per-request map was worth 22%. + */ + public interface Writable { + void writeTo(ByteSink out); + } + + /** + * One JSON string, escaped, straight into {@code out}. + * + * Public because generated code calls it. A codec that knows its field types + * at build time emits a direct call here instead of putting the value in a Map + * and letting {@link #write} rediscover its type at run time. + */ + public static void writeString(String value, ByteSink out) { + if(value == null) { + out.putAscii("null"); + return; + } + writeString(out, value); + } + + /** + * One JSON value of unknown type, for the cases a generated codec cannot + * resolve statically (an unmodelled java.* type, a heterogeneous collection). + * The generated code uses the typed calls wherever it can and falls back here + * only where it must. + */ + public static void writeValue(Object value, ByteSink out) { + writeValue(out, value); + } + + private static void writeValue(ByteSink out, Object value) { + if(value instanceof Writable) { + // Checked first: a Writable is a DTO with a generated writer, and the + // clauses below would otherwise fall through to its toString(). + ((Writable)value).writeTo(out); + return; + } + if(value == null) { + out.putAscii("null"); + return; + } + if(value instanceof String) { + writeString(out, (String)value); + return; + } + if(value instanceof Integer || value instanceof Long + || value instanceof Short || value instanceof Byte) { + out.putNumber(((Number)value).longValue()); + return; + } + if(value instanceof Boolean) { + out.putAscii(((Boolean)value).booleanValue() ? "true" : "false"); + return; + } + if(value instanceof Double || value instanceof Float) { + double d = ((Number)value).doubleValue(); + // JSON has no Infinity or NaN; emitting them produces a document no + // parser will read back. Same rule as the String form. + if(Double.isNaN(d) || Double.isInfinite(d)) { + out.putAscii("null"); + return; + } + // The VALUE's own spelling, not the widened one. Float 1.2f widened + // to double prints as 1.2000000476837158, while the String writer + // prints Float.toString and gets 1.2 -- so one handler answered two + // different numbers depending on which protocol the client happened + // to negotiate, since HTTP/1.1 writes through this sink and HTTP/2 + // through the other. The two have to agree; the byte[] branch below + // carries the same warning for the same reason. + out.putAscii(value.toString()); + return; + } + if(value instanceof Map) { + out.put('{'); + Map map = (Map)value; + java.util.Iterator it = map.keySet().iterator(); + boolean first = true; + while(it.hasNext()) { + Object key = it.next(); + if(!first) { + out.put(','); + } + first = false; + writeString(out, key == null ? "null" : String.valueOf(key)); + out.put(':'); + writeValue(out, map.get(key)); + } + out.put('}'); + return; + } + if(value instanceof List) { + out.put('['); + List list = (List)value; + for(int iter = 0 ; iter < list.size() ; iter++) { + if(iter > 0) { + out.put(','); + } + writeValue(out, list.get(iter)); + } + out.put(']'); + return; + } + if(value instanceof Collection) { + // A Set is a JSON array too. Falling through to the String branch below + // wrote its toString() as a quoted "[a, b]", which parses as a string and + // is silently the wrong shape rather than an error. Indexed above because + // a List answers get(i) without building an iterator. + out.put('['); + Iterator it = ((Collection)value).iterator(); + boolean first = true; + while(it.hasNext()) { + if(!first) { + out.put(','); + } + first = false; + writeValue(out, it.next()); + } + out.put(']'); + return; + } + if(value instanceof byte[]) { + writeString(out, Base64Url.encode((byte[])value)); + return; + } + writeString(out, String.valueOf(value)); + } + + private static void writeString(ByteSink out, String value) { + out.put('"'); + int n = value.length(); + for(int iter = 0 ; iter < n ; iter++) { + char c = value.charAt(iter); + switch(c) { + case '"': out.putAscii("\\\""); break; + case '\\': out.putAscii("\\\\"); break; + case '\n': out.putAscii("\\n"); break; + case '\r': out.putAscii("\\r"); break; + case '\t': out.putAscii("\\t"); break; + case '\b': out.putAscii("\\b"); break; + case '\f': out.putAscii("\\f"); break; + default: + if(c < 0x20) { + // Control characters must be escaped, and the six-character + // form is the only one JSON allows for those without a + // short escape. (Spelling it out rather than writing the + // escape prefix: Java expands that sequence inside + // COMMENTS too, and the file stops compiling.) + out.putAscii("\\u00"); + out.put(hexDigit((c >> 4) & 0xf)); + out.put(hexDigit(c & 0xf)); + } else if(c < 0x80) { + out.put(c); + } else if(c >= 0xd800 && c <= 0xdbff && iter + 1 < n + && value.charAt(iter + 1) >= 0xdc00 + && value.charAt(iter + 1) <= 0xdfff) { + // A surrogate PAIR is one code point. Encoding the halves + // separately produced two replacement characters -- an + // emoji came out as garbage -- which is what comparing + // this writer's bytes against the String form caught. + out.putCodePoint(0x10000 + ((c - 0xd800) << 10) + + (value.charAt(iter + 1) - 0xdc00)); + iter++; + } else if(c >= 0xd800 && c <= 0xdfff) { + // An unpaired surrogate has no UTF-8 form at all, so it + // cannot be written literally -- it has to be escaped or + // substituted. The escape is the only one of the two that + // loses nothing, and it is what the String form does too, + // so both writers stay byte for byte identical. + out.putAscii("\\u"); + out.put(hexDigit((c >> 12) & 0xf)); + out.put(hexDigit((c >> 8) & 0xf)); + out.put(hexDigit((c >> 4) & 0xf)); + out.put(hexDigit(c & 0xf)); + } else { + out.putCodePoint(c); + } + break; + } + } + out.put('"'); + } + + private static int hexDigit(int nibble) { + return nibble < 10 ? '0' + nibble : 'a' + (nibble - 10); + } + + private static void writeValue(StringBuilder out, Object value) { + if(value == null) { + out.append("null"); + return; + } + // Before the String branch, and for the same reason the sink writer checks + // it first: a Writable is a DTO carrying its own generated writer. Without + // this it reached the quoting branch below and was emitted as the JSON + // STRING of its toString(), so one handler returned an object over HTTP/1.1 + // and unusable text over HTTP/2 -- the two writers disagreeing about a + // value's type, exactly as they did over Short and Byte. + if(value instanceof Writable) { + ByteSink sink = new ByteSink(256); + ((Writable)value).writeTo(sink); + try { + out.append(new String(sink.bytes(), 0, sink.length(), "UTF-8")); + } catch (java.io.UnsupportedEncodingException never) { + // UTF-8 is required of every VM. + throw new IllegalStateException(never.toString()); + } + return; + } + if(value instanceof String) { + writeString(out, (String)value); + return; + } + // Short and Byte belong here with the other integral types. The ByteSink + // writer already treats them as numbers, and leaving them out here sent + // them to the quoting branch below, so Json.write(Short) produced "1" + // where the sink produced 1 -- the same value with a different JSON type + // depending on which writer the caller reached. + if(value instanceof Boolean || value instanceof Integer || value instanceof Long + || value instanceof Short || value instanceof Byte) { + out.append(value.toString()); + return; + } + if(value instanceof Double || value instanceof Float) { + double d = ((Number)value).doubleValue(); + // JSON has no Infinity or NaN; emitting them produces a document no + // parser will read back. + if(Double.isNaN(d) || Double.isInfinite(d)) { + out.append("null"); + } else { + out.append(value.toString()); + } + return; + } + if(value instanceof Map) { + Map map = (Map)value; + out.append('{'); + boolean first = true; + java.util.Iterator it = map.keySet().iterator(); + while(it.hasNext()) { + Object key = it.next(); + if(!first) { + out.append(','); + } + first = false; + writeString(out, String.valueOf(key)); + out.append(':'); + writeValue(out, map.get(key)); + } + out.append('}'); + return; + } + if(value instanceof List) { + List list = (List)value; + out.append('['); + for(int iter = 0 ; iter < list.size() ; iter++) { + if(iter > 0) { + out.append(','); + } + writeValue(out, list.get(iter)); + } + out.append(']'); + return; + } + if(value instanceof byte[]) { + // As base64url, which is what the byte-sink writer does. The two have to + // agree: HTTP/1.1 writes through the sink and HTTP/2 through this one, so + // a disagreement means a BLOB comes back readable over one protocol and + // as "[B@1a2b3c" over the other, from the same handler. + writeString(out, Base64Url.encode((byte[])value)); + return; + } + if(value instanceof Collection) { + // As above: the two writers have to agree on what a Set is. + out.append('['); + Iterator it = ((Collection)value).iterator(); + boolean first = true; + while(it.hasNext()) { + if(!first) { + out.append(','); + } + first = false; + writeValue(out, it.next()); + } + out.append(']'); + return; + } + writeString(out, value.toString()); + } + + private static void writeString(StringBuilder out, String value) { + out.append('"'); + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + switch(c) { + case '"': out.append("\\\""); break; + case '\\': out.append("\\\\"); break; + case '\n': out.append("\\n"); break; + case '\r': out.append("\\r"); break; + case '\t': out.append("\\t"); break; + case '\b': out.append("\\b"); break; + case '\f': out.append("\\f"); break; + default: + if(c < 0x20) { + String hex = Integer.toHexString(c); + out.append("\\u"); + for(int pad = hex.length() ; pad < 4 ; pad++) { + out.append('0'); + } + out.append(hex); + } else if(c >= 0xd800 && c <= 0xdbff && iter + 1 < value.length() + && value.charAt(iter + 1) >= 0xdc00 + && value.charAt(iter + 1) <= 0xdfff) { + out.append(c); + iter++; + out.append(value.charAt(iter)); + } else if(c >= 0xd800 && c <= 0xdfff) { + // Unpaired: appending it produces a String that no UTF-8 + // encoder can represent, so it silently became '?' on the + // wire. The escape keeps the value intact and matches what + // the byte writer emits. + out.append("\\u"); + out.append(Integer.toHexString(c)); + } else { + out.append(c); + } + } + } + out.append('"'); + } +} diff --git a/vm/backend/src/com/codename1/backend/Jwt.java b/vm/backend/src/com/codename1/backend/Jwt.java new file mode 100644 index 00000000000..0453b157c8d --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Jwt.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * HS256 JSON Web Tokens: issue one, and verify one you are handed. + * + * Only HS256 is accepted, deliberately. A verifier that reads the algorithm out of + * the token it is checking is the classic JWT hole - "alg":"none" then verifies + * anything, and "alg":"HS256" against an RSA public key turns a public value into + * the signing secret. The algorithm is a property of THIS verifier, not of the + * token, so the header's alg is checked for agreement and never used to select + * anything. + */ +public final class Jwt { + private static final String HEADER = "{\"alg\":\"HS256\",\"typ\":\"JWT\"}"; + + private Jwt() { + } + + /** Thrown for any token that is not valid and current. */ + public static final class InvalidTokenException extends IOException { + InvalidTokenException(String message) { + super(message); + } + } + + /** + * - `claims`: the payload; "iat" and "exp" are set here and overwrite anything + * the caller put there + * - `ttlSeconds`: how long the token is good for + */ + public static String issue(Map claims, byte[] secret, long ttlSeconds) throws IOException { + if(secret == null || secret.length < 32) { + // A short secret makes HS256 brute-forceable offline, and there is no + // reason to allow one when generating a good one is a single call. + throw new IOException("The signing secret must be at least 32 bytes"); + } + long now = System.currentTimeMillis() / 1000L; + Map payload = new LinkedHashMap(); + if(claims != null) { + payload.putAll(claims); + } + payload.put("iat", new Long(now)); + payload.put("exp", new Long(now + ttlSeconds)); + String signingInput = Base64Url.encode(Crypto.utf8(HEADER)) + + "." + Base64Url.encode(Crypto.utf8(Json.write(payload))); + byte[] mac = Crypto.hmacSha256(secret, Crypto.utf8(signingInput)); + if(mac == null) { + throw new IOException("Could not sign the token"); + } + return signingInput + "." + Base64Url.encode(mac); + } + + /** + * Returns the claims of a token that is well-formed, correctly signed and not + * expired. Throws otherwise; there is no "valid but expired" return. + */ + public static Map verify(String token, byte[] secret) throws IOException { + if(token == null || secret == null) { + throw new InvalidTokenException("No token"); + } + // The same floor issue() enforces. Verifying with a short secret is the + // dangerous half: an empty or guessable key lets anyone compute a valid + // HS256 signature over claims of their choosing, and this would have + // accepted it. Not an InvalidTokenException, because the token is not + // what is wrong -- the deployment is, and it should fail closed and say + // so rather than read as a client sending a bad token. + if(secret.length < 32) { + throw new IOException("The verification secret must be at least 32 " + + "bytes; a shorter one is forgeable"); + } + int firstDot = token.indexOf('.'); + int secondDot = firstDot < 0 ? -1 : token.indexOf('.', firstDot + 1); + if(firstDot <= 0 || secondDot <= firstDot || token.indexOf('.', secondDot + 1) >= 0) { + throw new InvalidTokenException("Malformed token"); + } + String signingInput = token.substring(0, secondDot); + byte[] provided = Base64Url.decode(token.substring(secondDot + 1)); + byte[] expected = Crypto.hmacSha256(secret, Crypto.utf8(signingInput)); + if(provided == null || expected == null + || !Crypto.equalsConstantTime(expected, provided)) { + // One message for every signature failure: distinguishing "bad + // signature" from "unknown key" tells an attacker which half to work on. + throw new InvalidTokenException("Bad signature"); + } + byte[] headerBytes = Base64Url.decode(token.substring(0, firstDot)); + byte[] payloadBytes = Base64Url.decode(token.substring(firstDot + 1, secondDot)); + if(headerBytes == null || payloadBytes == null) { + throw new InvalidTokenException("Malformed token"); + } + Map header; + Map payload; + try { + header = Json.parseObject(new String(headerBytes, "UTF-8")); + payload = Json.parseObject(new String(payloadBytes, "UTF-8")); + } catch (Exception err) { + throw new InvalidTokenException("Malformed token"); + } + // Checked for agreement, never used to choose an algorithm. + if(!"HS256".equals(header.get("alg"))) { + throw new InvalidTokenException("Unsupported algorithm"); + } + Object exp = payload.get("exp"); + if(!(exp instanceof Number)) { + throw new InvalidTokenException("Token has no expiry"); + } + if(((Number)exp).longValue() <= System.currentTimeMillis() / 1000L) { + throw new InvalidTokenException("Token expired"); + } + return payload; + } + + /** Pulls the token out of an "Authorization: Bearer ..." header. */ + public static String bearer(String authorizationHeader) { + if(authorizationHeader == null) { + return null; + } + String prefix = "bearer "; + // regionMatches(true, ...) rather than folding: it compares character by + // character and is LOCALE INDEPENDENT, where toLowerCase() is not. On a + // Turkish device "Bearer " folds to a dotless i and stops equalling this + // constant, so every bearer token is refused and the API rejects everyone + // with nothing thrown to say why. It allocates nothing either. + if(authorizationHeader.length() <= prefix.length() + || !authorizationHeader.regionMatches(true, 0, prefix, 0, prefix.length())) { + return null; + } + return authorizationHeader.substring(prefix.length()).trim(); + } +} diff --git a/vm/backend/src/com/codename1/backend/LambdaRuntime.java b/vm/backend/src/com/codename1/backend/LambdaRuntime.java new file mode 100644 index 00000000000..2b55eec0503 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/LambdaRuntime.java @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * The AWS Lambda custom-runtime loop. + * + * Why this is the first server-side target: the Lambda Runtime API is a + * CLIENT-side HTTP/1.1 poll over plaintext loopback, and the host serialises + * invocations one per instance. So a runtime needs no listening socket, no + * event loop, no TLS and no virtual threads - exactly the four things a general + * server runtime needs and this VM does not yet have. What it does need is fast + * start-up, which is what a translated binary is good at. + * + * Protocol (2018-06-01): long-poll GET .../invocation/next, which blocks until an + * invocation arrives and returns the payload plus a Lambda-Runtime-Aws-Request-Id + * header; then POST the result to .../invocation/{id}/response, or the failure to + * .../invocation/{id}/error. + */ +public final class LambdaRuntime { + private static final String API_VERSION = "/2018-06-01/runtime"; + private static final String REQUEST_ID_HEADER = "Lambda-Runtime-Aws-Request-Id"; + + private LambdaRuntime() { + } + + /** + * Runs the invocation loop until the process is killed, which is how a Lambda + * runtime is supposed to end - the host freezes or terminates the instance. + */ + public static void run(Handler handler) { + String endpoint = System.getenv("AWS_LAMBDA_RUNTIME_API"); + if(endpoint == null) { + System.err.println("AWS_LAMBDA_RUNTIME_API is not set; not running under a Lambda host"); + return; + } + String host = endpoint; + int port = 80; + int colon = endpoint.indexOf(':'); + if(colon > 0) { + host = endpoint.substring(0, colon); + try { + port = Integer.parseInt(endpoint.substring(colon + 1)); + } catch (NumberFormatException err) { + System.err.println("Malformed AWS_LAMBDA_RUNTIME_API: " + endpoint); + return; + } + } + while(true) { + if(!pumpOnce(handler, host, port)) { + return; + } + } + } + + /** + * One poll/dispatch/report cycle. Returns false when the loop should stop, + * which currently means the control connection itself failed - there is no + * useful recovery from that, and spinning would burn the instance's budget. + */ + static boolean pumpOnce(Handler handler, String host, int port) { + Http.Response next; + try { + next = Http.get(host, port, API_VERSION + "/invocation/next"); + } catch (Exception err) { + System.err.println("Failed to poll for the next invocation: " + err); + return false; + } + String requestId = next.getHeader(REQUEST_ID_HEADER); + if(requestId == null) { + System.err.println("Invocation carried no " + REQUEST_ID_HEADER + "; cannot report a result"); + return false; + } + String result; + try { + result = handler.handle(next.getBodyAsString(), requestId); + } catch (Exception err) { + // The same rule the response path below takes, and for the same + // reason: an invocation the host was never told about stays + // outstanding until it times out, and polling for another one while + // that is true just strands them one after the next. If the failure + // could not even be reported, nothing this process says is reaching + // the host, so it stops rather than collecting more. + if(!reportError(host, port, requestId, err)) { + System.err.println("The runtime API is unreachable, so this runtime is " + + "stopping rather than collecting invocations it cannot answer."); + return false; + } + return true; + } + try { + byte[] payload = (result == null ? "null" : result).getBytes("UTF-8"); + // The status matters: the Runtime API REJECTS a result it will not take + // -- 413 for a payload over the response limit is the ordinary case -- + // and answers rather than throwing. Discarding it meant the handler's + // work was dropped and the loop went straight back to polling, with the + // caller left waiting for a reply that was never accepted and nothing + // anywhere saying why. + Http.Response posted = Http.post(host, port, + API_VERSION + "/invocation/" + requestId + "/response", payload); + if(posted == null || posted.getStatus() < 200 || posted.getStatus() >= 300) { + System.err.println("The Lambda runtime API refused the response for " + + requestId + " with status " + + (posted == null ? "none" : String.valueOf(posted.getStatus())) + + "; the result of " + payload.length + " byte(s) was not " + + "delivered. Reporting it as an error so the invocation " + + "does not simply hang."); + // And stop if even THAT could not be delivered. The result is + // already gone, so an unreported invocation stays outstanding + // until the host times it out while this loop takes the next + // one. Third branch with this rule; they are the three ways an + // invocation can end without the host being told. + if(!reportError(host, port, requestId, new java.io.IOException( + "the runtime API refused the response with status " + + (posted == null ? "none" : String.valueOf(posted.getStatus()))))) { + System.err.println("The runtime API is unreachable, so this runtime is " + + "stopping rather than collecting invocations it cannot answer."); + return false; + } + } + } catch (Exception err) { + // The result is GONE -- it existed only in the request that just + // failed -- so this invocation has to be resolved here or it stays + // outstanding until the host times it out, while this loop cheerfully + // takes the next one. Reporting the failure is what lets the host + // fail it now instead. + System.err.println("Failed to post the response for " + requestId + ": " + err + + "; reporting it as an error so the invocation is resolved rather " + + "than left outstanding."); + if(!reportError(host, port, requestId, err)) { + // Not even the error reached the host, so nothing this process + // says is getting through. Stop polling: collecting further + // invocations only strands them the same way, and an exited + // runtime is something Lambda knows how to recover from. + System.err.println("The runtime API is unreachable, so this runtime is " + + "stopping rather than collecting invocations it cannot answer."); + return false; + } + } + return true; + } + + /** @return whether the host accepted the report, so a caller can stop. */ + private static boolean reportError(String host, int port, String requestId, Exception cause) { + try { + // The host parses this shape; a plain string body is reported as a + // malformed error and masks the real failure. + String json = "{\"errorType\":\"" + escape(cause.getClass().getName()) + + "\",\"errorMessage\":" + quote(cause.getMessage()) + "}"; + Http.Response posted = Http.post(host, port, + API_VERSION + "/invocation/" + requestId + "/error", + json.getBytes("UTF-8")); + // Nothing left to escalate to if even this is refused, but a silent + // failure here is how an invocation disappears without a trace. + if(posted == null || posted.getStatus() < 200 || posted.getStatus() >= 300) { + System.err.println("The Lambda runtime API refused the error report for " + + requestId + " with status " + + (posted == null ? "none" : String.valueOf(posted.getStatus()))); + return false; + } + return true; + } catch (Exception err) { + System.err.println("Failed to report the error for " + requestId + ": " + err); + return false; + } + } + + private static String quote(String value) { + if(value == null) { + return "null"; + } + return "\"" + escape(value) + "\""; + } + + private static String escape(String value) { + StringBuilder out = new StringBuilder(); + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + switch(c) { + case '"': + out.append("\\\""); + break; + case '\\': + out.append("\\\\"); + break; + case '\n': + out.append("\\n"); + break; + case '\r': + out.append("\\r"); + break; + case '\t': + out.append("\\t"); + break; + default: + if(c < 0x20) { + out.append("\\u").append(hex(c)); + } else { + out.append(c); + } + } + } + return out.toString(); + } + + private static String hex(char c) { + String h = Integer.toHexString(c); + StringBuilder out = new StringBuilder(); + for(int iter = h.length() ; iter < 4 ; iter++) { + out.append('0'); + } + return out.append(h).toString(); + } +} diff --git a/vm/backend/src/com/codename1/backend/StaticFiles.java b/vm/backend/src/com/codename1/backend/StaticFiles.java new file mode 100644 index 00000000000..3cd2e8435b9 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/StaticFiles.java @@ -0,0 +1,622 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Serves files out of a document root, on the kernel's zero-copy path. + * + * The body goes out with sendfile() where the platform has it: the bytes move from + * the page cache to the socket inside the kernel, never entering this process. For + * a file server that is the difference between two copies per byte and none. TLS + * is the exception and always will be -- encrypted bytes have to be produced in + * user space, so that path reads and writes like anything else. + * + * Correctness this does NOT cut corners on: + * + * - the resolved file must be inside the root, proven with realpath() rather + * than by inspecting the request string. "../" is only the obvious attack; + * percent-encoding and a symlink pointing out of the tree are the other two, + * and only resolution catches all three + * - the descriptor is opened FIRST and stat'd from the open fd, so the length in + * the header and the bytes in the body describe the same file even if it is + * replaced mid-request + * - conditional requests (If-None-Match, If-Modified-Since) and ranges, because + * a static server without them re-sends whole files to clients that already + * have them + */ +public final class StaticFiles implements HttpServer.Handler { + private static final boolean HAVE_SENDFILE = FileIo.hasSendFile(); + + private final String root; + private final String prefix; + private final String indexFile; + private final String cacheControl; + + /** + * - `root`: the document root; resolved once, and every request must land inside it + * - `prefix`: URL prefix to strip, "" or "/" for none + * - `cacheControl`: the Cache-Control value, or null to omit it + */ + public StaticFiles(String root, String prefix, String indexFile, String cacheControl) throws IOException { + String resolved = FileIo.realPath(root); + if(resolved == null) { + throw new IOException("Document root does not exist: " + root); + } + this.root = resolved; + this.prefix = prefix == null || "/".equals(prefix) ? "" : stripTrailingSlash(prefix); + this.indexFile = indexFile == null ? "index.html" : indexFile; + this.cacheControl = cacheControl; + } + + /** True when the body is sent by the kernel rather than copied through here. */ + public static boolean isZeroCopy() { + return HAVE_SENDFILE; + } + + public HttpServer.Response handle(HttpServer.Request request) throws Exception { + String target = request.getTarget(); + int q = target.indexOf('?'); + if(q >= 0) { + target = target.substring(0, q); + } + if(prefix.length() > 0) { + // The prefix has to end on a segment boundary. startsWith alone let + // /assets2/logo.png match a prefix of /assets, strip to /2/logo.png and + // be served from the document root, which is a different URL namespace + // than the one this handler was mounted on. + if(!target.startsWith(prefix) + || (target.length() > prefix.length() + && target.charAt(prefix.length()) != '/')) { + return null; // not ours; let the caller 404 it + } + target = target.substring(prefix.length()); + } + // The method is checked only once the target is known to be OURS. This + // handler is one link in a chain -- the caller tries it and falls back -- + // so refusing a verb for a path outside the mount answers on behalf of + // whoever was going to handle it: a POST to an unrelated path came back + // 405 instead of reaching the 404 the caller meant, and in a chain that + // tries files first it would shadow a later dynamic handler entirely. + String method = request.getMethod(); + if(!"GET".equalsIgnoreCase(method) && !"HEAD".equalsIgnoreCase(method)) { + return HttpServer.Response.text(405, "method not allowed"); + } + String decoded = decode(target); + if(decoded == null) { + return HttpServer.Response.text(400, "bad path"); + } + if(decoded.indexOf('\0') >= 0) { + // A NUL truncates the path in every C call underneath this. + return HttpServer.Response.text(400, "bad path"); + } + if(!decoded.startsWith("/")) { + decoded = "/" + decoded; + } + if(decoded.endsWith("/")) { + decoded = decoded + indexFile; + } + + // Open under the root so the kernel refuses an escape while it resolves. + // The realPath check further down runs against a SECOND lookup, so on its + // own it loses a race an attacker who can write symlinks into the document + // root controls: point the link outside for this open, inside for the + // check, and the descriptor served is the outside file. Where openBeneath + // works, containment is already settled by the time the descriptor exists. + boolean beneathProven = true; + int fd = FileIo.openBeneath(root, decoded); + if(fd == FileIo.BENEATH_UNSUPPORTED) { + beneathProven = false; + fd = FileIo.openRead(root + decoded); + } + if(fd < 0) { + return HttpServer.Response.text(404, "not found"); + } + boolean release = true; + try { + long[] info = new long[3]; + if(FileIo.stat(fd, info) != 0) { + return HttpServer.Response.text(404, "not found"); + } + if(info[2] != 0) { + // A directory: retry at its index file rather than listing it. + // Directory listings leak names nobody asked to publish. + FileIo.close(fd); + release = false; + String rawTarget = request.getTarget() == null ? "" : request.getTarget(); + int queryAt = rawTarget.indexOf('?'); + String rawPath = queryAt < 0 ? rawTarget : rawTarget.substring(0, queryAt); + if(!rawPath.endsWith("/")) { + // Redirect first. Serving the index at /static/docs makes a browser + // resolve "style.css" in it against /static/, not /static/docs/, so + // every relative reference in an otherwise valid site points one + // level too high. The query is carried across because it was + // addressed to this resource. + Map moved = new LinkedHashMap(); + moved.put("Location", rawPath + "/" + + (queryAt < 0 ? "" : rawTarget.substring(queryAt))); + return HttpServer.Response.empty(301, "text/plain", moved); + } + String indexPath = stripTrailingSlash(decoded) + "/" + indexFile; + int indexFd = beneathProven ? FileIo.openBeneath(root, indexPath) + : FileIo.openRead(root + indexPath); + if(indexFd == FileIo.BENEATH_UNSUPPORTED) { + beneathProven = false; + indexFd = FileIo.openRead(root + indexPath); + } + if(indexFd < 0) { + return HttpServer.Response.text(404, "not found"); + } + fd = indexFd; + release = true; + if(FileIo.stat(fd, info) != 0 || info[2] != 0) { + return HttpServer.Response.text(404, "not found"); + } + decoded = stripTrailingSlash(decoded) + "/" + indexFile; + } + + // Only where the open could not prove it. Checking the request string + // instead is defeated by an encoded traversal or by a symlink out of the + // tree, so this resolves first -- but it is a second lookup, which is why + // the open above is preferred wherever the platform supports it. + if(!beneathProven) { + String real = FileIo.realPath(root + decoded); + if(real == null || !isInsideRoot(real)) { + return HttpServer.Response.text(403, "forbidden"); + } + } + + long size = info[0]; + long modified = info[1]; + // JavaAPI's Long has neither toHexString nor a radix toString. An + // ETag only has to be stable and opaque, so size-mtime in decimal is + // exactly as good a validator. + String etag = "\"" + size + "-" + modified + "\""; + + Map headers = new LinkedHashMap(); + headers.put("ETag", etag); + headers.put("Last-Modified", Http1Date.format(modified)); + headers.put("Accept-Ranges", "bytes"); + if(cacheControl != null) { + headers.put("Cache-Control", cacheControl); + } + + if(isNotModified(request, etag, modified)) { + FileIo.close(fd); + release = false; + // 304 carries the validators and no body, by definition. + return HttpServer.Response.empty(304, contentType(decoded), headers); + } + + long offset = 0; + long length = size; + int status = 200; + String range = request.getHeader("range"); + if(range != null && rangeIsFresh(request, etag, modified)) { + long[] parsed = parseRange(range, size); + if(parsed == IGNORE_RANGE) { + // Nothing wrong with the request; this server just cannot + // answer it as a range. Send the representation whole. + parsed = null; + } else if(parsed == null) { + headers.put("Content-Range", "bytes */" + size); + FileIo.close(fd); + release = false; + return HttpServer.Response.empty(416, contentType(decoded), headers); + } + if(parsed != null) { + offset = parsed[0]; + length = parsed[1]; + status = 206; + headers.put("Content-Range", + "bytes " + offset + "-" + (offset + length - 1) + "/" + size); + } + } + + release = false; // the server owns the descriptor from here + return HttpServer.Response.file(status, contentType(decoded), trackFile(fd), offset, length, headers); + } finally { + if(release) { + FileIo.close(fd); + } + } + } + + private boolean isInsideRoot(String real) { + if(real.equals(root)) { + return true; + } + // The separator matters: "/srv/wwwroot-evil" starts with "/srv/www" but is + // not inside it. EITHER separator, though: FileIo.realPath answers + // backslashes on Windows, so requiring root + "/" refused every ordinary + // child there -- and since openBeneath() is unsupported in the Java SE + // runtime, every request in a Windows dev loop reaches this fallback and + // was answered 403. The packaged server is POSIX-only; the developer + // running cn1:backend is not. + String base = root; + if(base.endsWith("/") || base.endsWith("\\")) { + base = base.substring(0, base.length() - 1); + } + if(!real.startsWith(base) || real.length() <= base.length()) { + return false; + } + char next = real.charAt(base.length()); + return next == '/' || next == '\\'; + } + + /** + * True when a Range may be honoured: either the client sent no If-Range, or the + * validator it sent still describes this file. + * + * A resumed download sends back the validator it received with the first part. If + * the file has changed since, answering 206 out of the new one lets the client + * staple fresh bytes onto a stale prefix and call the result a complete download. + * HTTP's answer is to ignore the range and send the whole current representation, + * which costs one download and saves a corrupt file. + */ + private static boolean rangeIsFresh(HttpServer.Request request, String etag, long modified) { + String ifRange = request.getHeader("if-range"); + if(ifRange == null) { + return true; + } + String value = ifRange.trim(); + if(value.length() == 0) { + return false; + } + if(value.charAt(0) == '"') { + return value.equals(etag); + } + if(value.startsWith("W/") || value.startsWith("w/")) { + // If-Range requires a strong comparison, and a weak tag cannot supply one. + return false; + } + long parsed = Http1Date.parse(value); + // Second granularity on the wire, as in isNotModified. + return parsed >= 0 && parsed / 1000 == modified / 1000; + } + + private static boolean isNotModified(HttpServer.Request request, String etag, long modified) { + String ifNoneMatch = request.getHeader("if-none-match"); + if(ifNoneMatch != null) { + // An ETag match wins outright; a date is only consulted when there is + // no ETag to compare, as HTTP requires. + return ifNoneMatch.indexOf(etag) >= 0 || "*".equals(ifNoneMatch.trim()); + } + String ifModifiedSince = request.getHeader("if-modified-since"); + if(ifModifiedSince == null) { + return false; + } + long since = Http1Date.parse(ifModifiedSince); + // Second granularity on the wire, so compare at that resolution. + return since >= 0 && modified / 1000 <= since / 1000; + } + + /** Returns {offset, length}, or null when the range cannot be satisfied. */ + /** + * Returned when the Range field cannot be honoured but nothing about it is + * wrong: the whole representation is sent, with a 200, exactly as if the + * client had not asked. Distinct from null, which means every range asked + * for is unsatisfiable and 416 is the answer. + */ + static final long[] IGNORE_RANGE = new long[0]; + + static long[] parseRange(String header, long size) { + String value = header.trim(); + if(!value.startsWith("bytes=")) { + return IGNORE_RANGE; + } + value = value.substring("bytes=".length()); + if(value.indexOf(',') >= 0) { + // Multi-range needs a multipart/byteranges body, which this does not + // build. But NOT satisfying a range is not the same as the range being + // unsatisfiable, and 416 says the second: RFC 9110 15.5.17 is for the + // case where none of what was asked for exists, and "bytes=0-99,200-299" + // over a large enough file is entirely satisfiable -- this server simply + // will not assemble it. The rule for a Range that cannot be honoured is + // to IGNORE the field and send the whole representation, which every + // client understands, rather than to refuse a request that is correct. + return IGNORE_RANGE; + } + int dash = value.indexOf('-'); + if(dash < 0) { + // Not a byte-range-spec at all. RFC 9110 14.2 says to IGNORE a Range + // the server cannot parse, not to refuse the request over it -- 416 + // asserts that what was asked for does not exist, which is a claim + // this cannot make about a field it did not understand. + return IGNORE_RANGE; + } + String fromText = value.substring(0, dash).trim(); + String toText = value.substring(dash + 1).trim(); + if(size == 0) { + // No range over a zero-length representation can be satisfied, and the + // suffix form quietly produced one: "bytes=-1" clamped to a length of 0 + // and answered 206 with "Content-Range: bytes 0--1/0", which is not a + // header any client can read. 416 is the whole of the correct answer. + return null; + } + try { + if(fromText.length() == 0) { + // "-N" is the last N bytes. + long n = Long.parseLong(toText); + if(n <= 0) { + return null; + } + if(n > size) { + n = size; + } + return new long[]{size - n, n}; + } + long from = Long.parseLong(fromText); + if(from < 0 || from >= size) { + return null; + } + long to = toText.length() == 0 ? size - 1 : Long.parseLong(toText); + if(to >= size) { + to = size - 1; + } + if(to < from) { + return null; + } + return new long[]{from, to - from + 1}; + } catch (NumberFormatException err) { + // Digits that are not digits: unparseable, so ignored for the same + // reason as above rather than answered 416. + return IGNORE_RANGE; + } + } + + /** + * Writes length bytes of fileFd to the socket. Loops because sendfile may move + * less than asked, which is normal rather than an error. + */ + static void sendBody(int socketFd, long session, int fileFd, long offset, long length) throws IOException { + long remaining = length; + long position = offset; + // sendfile works because the kernel moves bytes it never has to look at. + // TLS bytes have to be encrypted in user space first, so there is no + // zero-copy path there and there never will be. + if(HAVE_SENDFILE && session == 0) { + while(remaining > 0) { + long sent = FileIo.sendFile(socketFd, fileFd, position, remaining); + if(sent < 0) { + throw new IOException("sendfile failed"); + } + if(sent == 0) { + // No progress and no error: the peer is gone. + throw new IOException("connection closed while sending"); + } + position += sent; + remaining -= sent; + } + return; + } + copyBody(socketFd, session, fileFd, offset, length); + } + + /** The read/write path: no sendfile on this platform, or the socket is TLS. */ + static void copyBody(int socketFd, long session, int fileFd, long offset, long length) throws IOException { + byte[] buffer = new byte[64 * 1024]; + long remaining = length; + // Only the plain path uses this with an offset; a fresh descriptor is at 0. + long skipped = 0; + while(skipped < offset) { + int want = (int)Math.min(buffer.length, offset - skipped); + int n = FileIo.read(fileFd, buffer, 0, want); + if(n <= 0) { + throw new IOException("Could not seek to the range start"); + } + skipped += n; + } + while(remaining > 0) { + int want = (int)Math.min(buffer.length, remaining); + int n = FileIo.read(fileFd, buffer, 0, want); + if(n <= 0) { + throw new IOException("Unexpected end of file while sending"); + } + if(session == 0) { + ServerSocket.write(socketFd, buffer, 0, n); + } else { + Tls.write(session, buffer, 0, n); + } + remaining -= n; + } + } + + /** + * Reads a range of the file into memory. Needed for HTTP/2, where the body has + * to become DATA frames and so cannot take the sendfile path. + */ + static byte[] readAll(int fd, long offset, long length) throws IOException { + if(length > Integer.MAX_VALUE) { + throw new IOException("File too large to buffer for HTTP/2"); + } + byte[] out = new byte[(int)length]; + byte[] skip = new byte[64 * 1024]; + long skipped = 0; + while(skipped < offset) { + int want = (int)Math.min(skip.length, offset - skipped); + int n = FileIo.read(fd, skip, 0, want); + if(n <= 0) { + throw new IOException("Could not seek to the range start"); + } + skipped += n; + } + int filled = 0; + while(filled < out.length) { + int n = FileIo.read(fd, out, filled, out.length - filled); + if(n <= 0) { + throw new IOException("Unexpected end of file"); + } + filled += n; + } + return out; + } + + /** + * Descriptors handed to a Response and not yet closed. + * + * Telemetry, and the only way a leak here is visible at all: a descriptor + * that escapes is not counted by anything else, the process limit is in the + * hundreds of thousands, and the failure arrives much later as a server that + * cannot accept sockets. An HTTP/2 HEAD of a static file leaked one per + * request precisely because nothing said so. + */ + private static final java.util.concurrent.atomic.AtomicInteger OPEN_FILES = + new java.util.concurrent.atomic.AtomicInteger(); + + static int openFileCount() { + return OPEN_FILES.get(); + } + + /** Counted where the descriptor becomes a Response's to own. */ + static int trackFile(int fd) { + if(fd >= 0) { + OPEN_FILES.incrementAndGet(); + } + return fd; + } + + /** + * Gives up tracking WITHOUT closing: the HTTP/2 session owns this descriptor + * now and frees it natively, so counting it here would climb for ever. + * + * The count means "descriptors this side still has to close" -- anything the + * session holds is reported separately by Http2.pendingBodyFiles(). Mixing + * the two made an ordinary h2 GET look like a leak, which is how this + * distinction got noticed. + */ + static void handOverFile(int fd) { + if(fd >= 0) { + OPEN_FILES.decrementAndGet(); + } + } + + static void closeFile(int fd) { + if(fd >= 0) { + OPEN_FILES.decrementAndGet(); + FileIo.close(fd); + } + } + + private static String stripTrailingSlash(String value) { + return value.length() > 1 && value.endsWith("/") + ? value.substring(0, value.length() - 1) : value; + } + + /** Null for a malformed escape rather than a partially decoded path. */ + static String decode(String value) { + if(value.indexOf('%') < 0) { + return value; + } + // A run of escapes is one UTF-8 sequence, not one character per octet. + // Appending each octet as a char turned the %C3%A9 a client sends for an + // accented letter into two characters, so the lookup missed a file that is + // on disk and the request 404'd. + StringBuilder out = new StringBuilder(); + byte[] pending = new byte[value.length()]; + int pendingLength = 0; + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + if(c != '%') { + if(pendingLength > 0) { + out.append(utf8(pending, pendingLength)); + pendingLength = 0; + } + out.append(c); + continue; + } + if(iter + 2 >= value.length()) { + return null; + } + try { + pending[pendingLength++] = + (byte)Integer.parseInt(value.substring(iter + 1, iter + 3), 16); + } catch (NumberFormatException err) { + return null; + } + iter += 2; + } + if(pendingLength > 0) { + out.append(utf8(pending, pendingLength)); + } + return out.toString(); + } + + /** The gathered escape bytes as text; malformed input keeps its bytes. */ + private static String utf8(byte[] bytes, int length) { + try { + return new String(bytes, 0, length, "UTF-8"); + } catch (java.io.UnsupportedEncodingException err) { + return new String(bytes, 0, length); + } + } + + /** + * ASCII lower case, because String.toLowerCase() is LOCALE SENSITIVE and + * this platform has no Locale to ask for the root one. On a device set to + * Turkish the I of an ASCII token folds to a dotless i, so the result stops + * equalling the constant it is compared against: nothing is thrown, nothing + * is logged, and the feature is simply inert for those users. Every token + * folded here -- a header name, a file extension -- is ASCII by + * specification. Copied rather than shared; see CLAUDE.md. + */ + private static String asciiLower(String value) { + if(value == null) { + return null; + } + StringBuilder out = new StringBuilder(value.length()); + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + out.append(c >= 'A' && c <= 'Z' ? (char)(c + 32) : c); + } + return out.toString(); + } + + static String contentType(String path) { + int dot = path.lastIndexOf('.'); + String ext = dot < 0 ? "" : asciiLower(path.substring(dot + 1)); + if("html".equals(ext) || "htm".equals(ext)) return "text/html; charset=utf-8"; + if("css".equals(ext)) return "text/css; charset=utf-8"; + if("js".equals(ext) || "mjs".equals(ext)) return "text/javascript; charset=utf-8"; + if("json".equals(ext)) return "application/json; charset=utf-8"; + if("svg".equals(ext)) return "image/svg+xml"; + if("png".equals(ext)) return "image/png"; + if("jpg".equals(ext) || "jpeg".equals(ext)) return "image/jpeg"; + if("gif".equals(ext)) return "image/gif"; + if("webp".equals(ext)) return "image/webp"; + if("ico".equals(ext)) return "image/x-icon"; + if("woff2".equals(ext)) return "font/woff2"; + if("woff".equals(ext)) return "font/woff"; + if("ttf".equals(ext)) return "font/ttf"; + if("wasm".equals(ext)) return "application/wasm"; + if("pdf".equals(ext)) return "application/pdf"; + if("txt".equals(ext) || "md".equals(ext)) return "text/plain; charset=utf-8"; + if("xml".equals(ext)) return "application/xml"; + if("mp4".equals(ext)) return "video/mp4"; + if("zip".equals(ext)) return "application/zip"; + return "application/octet-stream"; + } +} diff --git a/vm/backend/src/com/codename1/backend/Utf8.java b/vm/backend/src/com/codename1/backend/Utf8.java new file mode 100644 index 00000000000..f4104273d9b --- /dev/null +++ b/vm/backend/src/com/codename1/backend/Utf8.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend; + +/** + * Whether a byte range really is UTF-8. + * + * `new String(bytes, "UTF-8")` never fails: a malformed sequence becomes U+FFFD + * and the caller is handed text the client did not send. For a request body that + * turns a protocol error into silent corruption -- the JSON parses, the handler + * runs, and whatever was validated was validated against the REPLACEMENT, not + * against what arrived. So the bytes are checked before they are decoded, and a + * body that is not UTF-8 is answered with a 400. + * + * Written out by hand because there is no CharsetDecoder here: vm/JavaAPI has + * Charset and StandardCharsets and nothing else, so CodingErrorAction.REPORT -- + * the way this is normally done -- does not exist on the target. + * + * The table is RFC 3629's, which is narrower than "any sequence that decodes": + * an overlong encoding, a surrogate half and anything above U+10FFFF are all + * rejected, because each of them is a way of spelling a character twice and the + * second spelling is what slips past a filter that only checked the first. + */ +final class Utf8 { + private Utf8() { + } + + static boolean isValid(byte[] bytes, int offset, int length) { + int at = offset; + int end = offset + length; + while(at < end) { + int first = bytes[at] & 0xff; + if(first < 0x80) { + at++; + continue; + } + int following; + int lowest; + int highest; + if(first >= 0xc2 && first <= 0xdf) { + following = 1; + lowest = 0x80; + highest = 0xbf; + } else if(first == 0xe0) { + // A second byte below A0 would be an overlong two-byte value. + following = 2; + lowest = 0xa0; + highest = 0xbf; + } else if(first >= 0xe1 && first <= 0xec) { + following = 2; + lowest = 0x80; + highest = 0xbf; + } else if(first == 0xed) { + // ED A0..BF is the surrogate range, which UTF-8 does not encode. + following = 2; + lowest = 0x80; + highest = 0x9f; + } else if(first == 0xee || first == 0xef) { + following = 2; + lowest = 0x80; + highest = 0xbf; + } else if(first == 0xf0) { + // Below 90 is an overlong three-byte value. + following = 3; + lowest = 0x90; + highest = 0xbf; + } else if(first >= 0xf1 && first <= 0xf3) { + following = 3; + lowest = 0x80; + highest = 0xbf; + } else if(first == 0xf4) { + // F4 90 and above is past U+10FFFF. + following = 3; + lowest = 0x80; + highest = 0x8f; + } else { + // 80..C1 is a continuation with nothing to continue, or an + // overlong one-byte form; F5..FF encodes nothing at all. + return false; + } + if(at + following >= end) { + return false; // truncated at the end of the range + } + int second = bytes[at + 1] & 0xff; + if(second < lowest || second > highest) { + return false; + } + for(int iter = 2 ; iter <= following ; iter++) { + int next = bytes[at + iter] & 0xff; + if(next < 0x80 || next > 0xbf) { + return false; + } + } + at += following + 1; + } + return true; + } +} diff --git a/vm/backend/src/com/codename1/backend/annotations/DeleteMapping.java b/vm/backend/src/com/codename1/backend/annotations/DeleteMapping.java new file mode 100644 index 00000000000..563b1930853 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/DeleteMapping.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Maps an HTTP DELETE to this method. +/// +/// The path may carry `{name}` segments, bound with [PathVariable]. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.METHOD) +public @interface DeleteMapping { + /// The path, relative to any [RequestMapping] on the class. + String[] value() default {}; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/Generated.java b/vm/backend/src/com/codename1/backend/annotations/Generated.java new file mode 100644 index 00000000000..7ab1218c3ad --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/Generated.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks a class the backend processors wrote, so a later pass knows its own work. + * + * Both processors refuse to generate a class whose name is already taken, because + * the generated one would silently overwrite the developer's in the output + * directory. That guard read its OWN previous output as such a class: an + * incremental build -- `mvn process-classes` a second time, without a clean -- + * scans target/classes, finds the router or dispatcher written by the first pass, + * and reports a collision. Every project using @RestController failed its second + * build and only a clean would fix it. + * + * CLASS retention: the class file has to carry it so the next pass's scanner can + * see it, and nothing reads it at runtime. + */ +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.TYPE) +public @interface Generated { +} diff --git a/vm/backend/src/com/codename1/backend/annotations/GetMapping.java b/vm/backend/src/com/codename1/backend/annotations/GetMapping.java new file mode 100644 index 00000000000..50166fd83f5 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/GetMapping.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Maps an HTTP GET to this method. +/// +/// The path may carry `{name}` segments, bound with [PathVariable]. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.METHOD) +public @interface GetMapping { + /// The path, relative to any [RequestMapping] on the class. + String[] value() default {}; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/PatchMapping.java b/vm/backend/src/com/codename1/backend/annotations/PatchMapping.java new file mode 100644 index 00000000000..39fe4c78fa7 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/PatchMapping.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Maps an HTTP PATCH to this method. +/// +/// The path may carry `{name}` segments, bound with [PathVariable]. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.METHOD) +public @interface PatchMapping { + /// The path, relative to any [RequestMapping] on the class. + String[] value() default {}; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/PathVariable.java b/vm/backend/src/com/codename1/backend/annotations/PathVariable.java new file mode 100644 index 00000000000..d997f60bd0d --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/PathVariable.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Binds a `{name}` segment of the path to this parameter. +/// +/// A missing value binds to the default below, or to null when the parameter is +/// not required. It is never a server error unless `required` says so. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.PARAMETER) +public @interface PathVariable { + /// The name to bind from. REQUIRED, and deliberately so: it cannot default to + /// the parameter's own name because a Java parameter name only survives + /// compilation when the application is built with -parameters, which is the + /// application's build to decide and not this one's. An annotation that + /// promised the default would compile fine and then fail at packaging, for + /// every developer who took it at its word. + String value(); + /// Whether a request without it is rejected. + boolean required() default true; + /// Used when the request omits it. + String defaultValue() default ""; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/PostMapping.java b/vm/backend/src/com/codename1/backend/annotations/PostMapping.java new file mode 100644 index 00000000000..2fe2db7fac4 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/PostMapping.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Maps an HTTP POST to this method. +/// +/// The path may carry `{name}` segments, bound with [PathVariable]. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.METHOD) +public @interface PostMapping { + /// The path, relative to any [RequestMapping] on the class. + String[] value() default {}; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/PutMapping.java b/vm/backend/src/com/codename1/backend/annotations/PutMapping.java new file mode 100644 index 00000000000..1f02608c0bd --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/PutMapping.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Maps an HTTP PUT to this method. +/// +/// The path may carry `{name}` segments, bound with [PathVariable]. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.METHOD) +public @interface PutMapping { + /// The path, relative to any [RequestMapping] on the class. + String[] value() default {}; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/RequestBody.java b/vm/backend/src/com/codename1/backend/annotations/RequestBody.java new file mode 100644 index 00000000000..a44021ba39d --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/RequestBody.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Binds the decoded request body to this parameter. +/// +/// The body is decoded to the parameter's declared type using the generated +/// codec for it, so a controller receives its own type rather than a Map. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.PARAMETER) +public @interface RequestBody { + /// Whether a request with no body is rejected. + boolean required() default true; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/RequestHeader.java b/vm/backend/src/com/codename1/backend/annotations/RequestHeader.java new file mode 100644 index 00000000000..15b71f58b1b --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/RequestHeader.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Binds a request header to this parameter. +/// +/// A missing value binds to the default below, or to null when the parameter is +/// not required. It is never a server error unless `required` says so. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.PARAMETER) +public @interface RequestHeader { + /// The name to bind from. REQUIRED, and deliberately so: it cannot default to + /// the parameter's own name because a Java parameter name only survives + /// compilation when the application is built with -parameters, which is the + /// application's build to decide and not this one's. An annotation that + /// promised the default would compile fine and then fail at packaging, for + /// every developer who took it at its word. + String value(); + /// Whether a request without it is rejected. + boolean required() default true; + /// Used when the request omits it. + String defaultValue() default ""; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/RequestMapping.java b/vm/backend/src/com/codename1/backend/annotations/RequestMapping.java new file mode 100644 index 00000000000..8aeef9d4f21 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/RequestMapping.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// A path, and optionally a verb, for a controller or one of its methods. +/// +/// On a class it prefixes every mapping inside it. On a method it is the general +/// form of the verb-specific annotations beside it, for verbs they do not cover. +@Retention(RetentionPolicy.CLASS) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface RequestMapping { + /// The path, or paths, this mapping answers on. + String[] value() default {}; + /// The HTTP verb, when this is used on a method directly. + String method() default ""; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/RequestParam.java b/vm/backend/src/com/codename1/backend/annotations/RequestParam.java new file mode 100644 index 00000000000..62b142425c1 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/RequestParam.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Binds a query parameter to this parameter. +/// +/// A missing value binds to the default below, or to null when the parameter is +/// not required. It is never a server error unless `required` says so. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.PARAMETER) +public @interface RequestParam { + /// The name to bind from. REQUIRED, and deliberately so: it cannot default to + /// the parameter's own name because a Java parameter name only survives + /// compilation when the application is built with -parameters, which is the + /// application's build to decide and not this one's. An annotation that + /// promised the default would compile fine and then fail at packaging, for + /// every developer who took it at its word. + String value(); + /// Whether a request without it is rejected. + boolean required() default true; + /// Used when the request omits it. + String defaultValue() default ""; +} diff --git a/vm/backend/src/com/codename1/backend/annotations/ResponseStatus.java b/vm/backend/src/com/codename1/backend/annotations/ResponseStatus.java new file mode 100644 index 00000000000..c76bb6f6643 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/ResponseStatus.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// The status this method answers with when it returns normally. +/// +/// Without it a method that returns a value answers 200, and a void method +/// answers 204. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.METHOD) +public @interface ResponseStatus { + /// The HTTP status code. + int value(); +} diff --git a/vm/backend/src/com/codename1/backend/annotations/RestController.java b/vm/backend/src/com/codename1/backend/annotations/RestController.java new file mode 100644 index 00000000000..0951f51c447 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/annotations/RestController.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/// Marks a class whose methods answer HTTP requests. +/// +/// The build scans for these and generates a router, so a controller is an +/// ordinary class with no base type, no interface, and nothing to register at +/// start-up. Deliberately the name Spring uses: the shape is meant to be +/// readable without learning it first. +/// +/// Routing is decided at BUILD time rather than by scanning at start-up or by a +/// map lookup per request. The generated router compares the target's bytes +/// where the parser already holds them, so a controller costs no more than the +/// hand-written equals chain it replaces. +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.TYPE) +public @interface RestController { +} diff --git a/vm/backend/src/com/codename1/backend/aws/Aws.java b/vm/backend/src/com/codename1/backend/aws/Aws.java new file mode 100644 index 00000000000..a3719d5a961 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/aws/Aws.java @@ -0,0 +1,397 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.aws; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import com.codename1.backend.Base64; +import com.codename1.backend.Crypto; +import com.codename1.backend.Web; + +/** + * AWS Signature Version 4, and the request plumbing every AWS service shares. + * + * This is the whole of what an AWS client needs that is not service specific: + * canonicalise a request, derive a signing key, and send it. {@link S3} is one + * service built on it; SQS, DynamoDB and Secrets Manager are the same three steps + * with a different host and payload, which is why the signer is a separate class + * rather than something private to S3. + * + * Written rather than pulled in because the AWS SDK is not an option here: it + * wants reflection, a class loader and a threading model a translated server + * binary does not have. SigV4 itself is a hash chain -- five HMACs and a SHA-256 + * -- over a canonical form of the request, and the specification is public and + * stable. + * + * The part that is easy to get wrong, and the reason for the length of this file, + * is the CANONICAL form: the signature covers a normalised URI, a sorted query + * string, sorted lower-cased headers and a hash of the body, and a single + * difference from what the service computes produces a 403 with no indication of + * which field disagreed. + */ +public final class Aws { + /** The unsigned-payload marker, for a body the caller does not want hashed. */ + public static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; + + private static final String ALGORITHM = "AWS4-HMAC-SHA256"; + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + private Aws() { + } + + /** + * A signed, sent request. + * + * `headers` are extra "Name: value" strings; Host, x-amz-date and + * x-amz-content-sha256 are added here because they are part of the signature. + */ + public static Web.Result send(Credentials credentials, String region, String service, + String method, String host, String path, Map query, Map headers, + byte[] body, String timestamp) throws IOException { + return send(credentials, region, service, method, host, path, query, headers, + body, timestamp, true); + } + + /** + * As above, over plain HTTP when `secure` is false. + * + * The signature covers the host and not the scheme, so this changes only how + * the request travels. It exists for a local MinIO or a test double on + * loopback; nothing reachable off the machine should use it, and AWS itself + * does not accept it. + */ + public static Web.Result send(Credentials credentials, String region, String service, + String method, String host, String path, Map query, Map headers, + byte[] body, String timestamp, boolean secure) throws IOException { + Map signedHeaders = headers == null ? new LinkedHashMap() : new LinkedHashMap(headers); + String stamp = timestamp == null ? Clock.timestamp() : timestamp; + String payloadHash = body == null ? sha256Hex(new byte[0]) : sha256Hex(body); + + signedHeaders.put("host", host); + signedHeaders.put("x-amz-date", stamp); + signedHeaders.put("x-amz-content-sha256", payloadHash); + if(credentials.getSessionToken() != null) { + // A temporary credential's token is part of the signature, not an + // afterthought: a request signed without it is rejected. + signedHeaders.put("x-amz-security-token", credentials.getSessionToken()); + } + + String authorization = authorization(credentials, region, service, method, path, + query, signedHeaders, payloadHash, stamp); + signedHeaders.put("authorization", authorization); + + List headerLines = new ArrayList(); + Iterator it = signedHeaders.entrySet().iterator(); + while(it.hasNext()) { + Map.Entry entry = (Map.Entry)it.next(); + headerLines.add(entry.getKey() + ": " + entry.getValue()); + } + String url = (secure ? "https://" : "http://") + host + encodePath(path); + String canonicalQuery = canonicalQuery(query); + if(canonicalQuery.length() > 0) { + url = url + "?" + canonicalQuery; + } + return Web.request(method, url, headerLines, body); + } + + /** The Authorization header value for one request. */ + public static String authorization(Credentials credentials, String region, String service, + String method, String path, Map query, Map headers, String payloadHash, + String timestamp) throws IOException { + String date = timestamp.substring(0, 8); + String scope = date + "/" + region + "/" + service + "/aws4_request"; + + // Header names are lower-cased and sorted; values have their runs of + // whitespace collapsed. All three are part of the specification, and all + // three are invisible in a failure -- the service just says 403. + TreeMap canonicalHeaders = new TreeMap(); + Iterator it = headers.entrySet().iterator(); + while(it.hasNext()) { + Map.Entry entry = (Map.Entry)it.next(); + canonicalHeaders.put(asciiLower(String.valueOf(entry.getKey())), + collapse(String.valueOf(entry.getValue()))); + } + StringBuilder headerBlock = new StringBuilder(); + StringBuilder signedNames = new StringBuilder(); + it = canonicalHeaders.entrySet().iterator(); + while(it.hasNext()) { + Map.Entry entry = (Map.Entry)it.next(); + headerBlock.append(entry.getKey()).append(':').append(entry.getValue()).append('\n'); + if(signedNames.length() > 0) { + signedNames.append(';'); + } + signedNames.append(entry.getKey()); + } + + String canonicalRequest = method + "\n" + + encodePath(path) + "\n" + + canonicalQuery(query) + "\n" + + headerBlock + "\n" + + signedNames + "\n" + + payloadHash; + String stringToSign = ALGORITHM + "\n" + timestamp + "\n" + scope + "\n" + + sha256Hex(utf8(canonicalRequest)); + byte[] signingKey = signingKey(credentials.getSecretKey(), date, region, service); + String signature = hex(Crypto.hmacSha256(signingKey, utf8(stringToSign))); + + return ALGORITHM + " Credential=" + credentials.getAccessKeyId() + "/" + scope + + ", SignedHeaders=" + signedNames + ", Signature=" + signature; + } + + /** + * A presigned URL: the signature travels in the query string, so anyone + * holding the URL can make that one request until it expires. + * + * This is what hands a mobile client a direct download or upload without + * proxying the bytes through the server, which is most of the reason to use + * object storage from an app at all. + */ + /** SigV4's ceiling: seven days. */ + private static final int MAX_PRESIGN_SECONDS = 7 * 24 * 60 * 60; + + public static String presign(Credentials credentials, String region, String service, + String method, String host, String path, Map query, int expiresSeconds, + String timestamp) throws IOException { + return presign(credentials, region, service, method, host, path, query, + expiresSeconds, timestamp, true); + } + + /** As above, producing an http:// URL when `secure` is false. See {@link #send}. */ + public static String presign(Credentials credentials, String region, String service, + String method, String host, String path, Map query, int expiresSeconds, + String timestamp, boolean secure) throws IOException { + // SigV4 accepts 1 second to 7 days, and anything else produces a URL that + // LOOKS right and is refused when someone tries to use it. These URLs are + // handed straight to a device, so the failure would surface far from the + // call that caused it -- and a caller computing a lifetime from + // configuration is exactly how a zero or a negative one gets here. + if(expiresSeconds < 1 || expiresSeconds > MAX_PRESIGN_SECONDS) { + throw new IOException("A presigned URL lasts between 1 second and 7 days; " + + expiresSeconds + " would be refused when it was used"); + } + String stamp = timestamp == null ? Clock.timestamp() : timestamp; + String date = stamp.substring(0, 8); + String scope = date + "/" + region + "/" + service + "/aws4_request"; + + Map signedQuery = query == null ? new LinkedHashMap() : new LinkedHashMap(query); + signedQuery.put("X-Amz-Algorithm", ALGORITHM); + signedQuery.put("X-Amz-Credential", credentials.getAccessKeyId() + "/" + scope); + signedQuery.put("X-Amz-Date", stamp); + signedQuery.put("X-Amz-Expires", String.valueOf(expiresSeconds)); + signedQuery.put("X-Amz-SignedHeaders", "host"); + if(credentials.getSessionToken() != null) { + signedQuery.put("X-Amz-Security-Token", credentials.getSessionToken()); + } + + String canonicalRequest = method + "\n" + + encodePath(path) + "\n" + + canonicalQuery(signedQuery) + "\n" + + "host:" + host + "\n\n" + + "host\n" + + UNSIGNED_PAYLOAD; + String stringToSign = ALGORITHM + "\n" + stamp + "\n" + scope + "\n" + + sha256Hex(utf8(canonicalRequest)); + byte[] signingKey = signingKey(credentials.getSecretKey(), date, region, service); + String signature = hex(Crypto.hmacSha256(signingKey, utf8(stringToSign))); + + signedQuery.put("X-Amz-Signature", signature); + return (secure ? "https://" : "http://") + host + encodePath(path) + "?" + + canonicalQuery(signedQuery); + } + + /** + * The four-step key derivation. The signing key is scoped to a date, a region + * and a service, which is what keeps a leaked signature from being reusable + * anywhere else. + * + * Public, along with the four canonicalisation helpers below, because a + * service this class does not wrap -- SQS, DynamoDB, Secrets Manager -- is the + * same signature over a different payload, and because these are the pieces a + * known-answer test can pin. A signature implementation that can only be + * tested end to end is one whose failures all look like 403. + */ + public static byte[] signingKey(String secretKey, String date, String region, String service) + throws IOException { + byte[] key = Crypto.hmacSha256(utf8("AWS4" + secretKey), utf8(date)); + key = Crypto.hmacSha256(key, utf8(region)); + key = Crypto.hmacSha256(key, utf8(service)); + return Crypto.hmacSha256(key, utf8("aws4_request")); + } + + /** + * Query parameters sorted by name, each name and value percent-encoded. + * Sorting is by the ENCODED name, which matters for names that differ only in + * a character the encoding changes. + */ + public static String canonicalQuery(Map query) { + if(query == null || query.isEmpty()) { + return ""; + } + List pairs = new ArrayList(); + Iterator it = query.entrySet().iterator(); + while(it.hasNext()) { + Map.Entry entry = (Map.Entry)it.next(); + Object value = entry.getValue(); + pairs.add(encode(String.valueOf(entry.getKey())) + "=" + + encode(value == null ? "" : String.valueOf(value))); + } + Collections.sort(pairs); + StringBuilder out = new StringBuilder(); + for(int iter = 0 ; iter < pairs.size() ; iter++) { + if(iter > 0) { + out.append('&'); + } + out.append(pairs.get(iter)); + } + return out.toString(); + } + + /** + * The path, percent-encoded segment by segment. The slashes between segments + * are NOT encoded; everything else that is not unreserved is -- which is why + * this cannot just call {@link #encode} on the whole path. + */ + public static String encodePath(String path) { + if(path == null || path.length() == 0) { + return "/"; + } + StringBuilder out = new StringBuilder(); + int at = 0; + while(at <= path.length()) { + int end = path.indexOf('/', at); + if(end < 0) { + end = path.length(); + } + out.append(encode(path.substring(at, end))); + if(end == path.length()) { + break; + } + out.append('/'); + at = end + 1; + } + return out.length() == 0 ? "/" : out.toString(); + } + + /** + * RFC 3986 unreserved characters pass; everything else becomes %XX with UPPER + * case hex. Note this is not URLEncoder: a space is %20 here, never '+', and + * '~' is not encoded. Both differences produce a signature mismatch. + */ + public static String encode(String value) { + if(value == null) { + return ""; + } + byte[] raw = utf8(value); + StringBuilder out = new StringBuilder(raw.length); + for(int iter = 0 ; iter < raw.length ; iter++) { + int c = raw[iter] & 0xff; + if((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') + || c == '-' || c == '_' || c == '.' || c == '~') { + out.append((char)c); + } else { + out.append('%') + .append(Character.toUpperCase(HEX[(c >> 4) & 0xf])) + .append(Character.toUpperCase(HEX[c & 0xf])); + } + } + return out.toString(); + } + + /** Leading and trailing space removed, internal runs collapsed to one space. */ + /** + * ASCII lower case, because String.toLowerCase() is LOCALE SENSITIVE and + * this platform has no Locale to ask for the root one. On a device set to + * Turkish the I of an ASCII token folds to a dotless i, so the result stops + * equalling the constant it is compared against: nothing is thrown, nothing + * is logged, and the feature is simply inert for those users. A header name + * is ASCII by specification. Copied rather than shared; see CLAUDE.md. + */ + private static String asciiLower(String value) { + if(value == null) { + return null; + } + StringBuilder out = new StringBuilder(value.length()); + for(int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + out.append(c >= 'A' && c <= 'Z' ? (char)(c + 32) : c); + } + return out.toString(); + } + + public static String collapse(String value) { + if(value == null) { + return ""; + } + StringBuilder out = new StringBuilder(); + boolean space = false; + String trimmed = value.trim(); + for(int iter = 0 ; iter < trimmed.length() ; iter++) { + char c = trimmed.charAt(iter); + if(c == ' ' || c == '\t') { + space = true; + continue; + } + if(space && out.length() > 0) { + out.append(' '); + } + space = false; + out.append(c); + } + return out.toString(); + } + + public static String sha256Hex(byte[] data) { + return hex(Crypto.sha256(data)); + } + + public static String hex(byte[] data) { + StringBuilder out = new StringBuilder(data.length * 2); + for(int iter = 0 ; iter < data.length ; iter++) { + out.append(HEX[(data[iter] >> 4) & 0xf]).append(HEX[data[iter] & 0xf]); + } + return out.toString(); + } + + static byte[] utf8(String value) { + if(value == null) { + return new byte[0]; + } + try { + return value.getBytes("UTF-8"); + } catch (java.io.UnsupportedEncodingException err) { + throw new IllegalStateException("UTF-8 is missing"); + } + } + + /** Base64 of a raw digest, for the services that want it that way. */ + static String base64(byte[] data) { + return Base64.encode(data); + } +} diff --git a/vm/backend/src/com/codename1/backend/aws/Clock.java b/vm/backend/src/com/codename1/backend/aws/Clock.java new file mode 100644 index 00000000000..520fcc633eb --- /dev/null +++ b/vm/backend/src/com/codename1/backend/aws/Clock.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.aws; + +/** + * The ISO basic timestamp AWS signs with: yyyyMMdd'T'HHmmss'Z', always UTC. + * + * Written out by hand rather than through SimpleDateFormat because the translated + * runtime's date formatting is locale-aware and this format must not be -- an + * Arabic-Indic digit or a locale that renders the year differently produces a + * signature the service cannot reproduce, and the only symptom is a 403. + */ +final class Clock { + private static final int[] DAYS_IN_MONTH = + {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + + private Clock() { + } + + static String timestamp() { + return timestamp(System.currentTimeMillis()); + } + + static String timestamp(long millis) { + long seconds = millis / 1000L; + if(millis < 0 && (millis % 1000L) != 0) { + seconds--; // floor, so a pre-epoch instant does not round toward zero + } + long days = floorDiv(seconds, 86400L); + int secondOfDay = (int)(seconds - days * 86400L); + + int year = 1970; + while(true) { + int length = isLeap(year) ? 366 : 365; + if(days >= length) { + days -= length; + year++; + } else if(days < 0) { + year--; + days += isLeap(year) ? 366 : 365; + } else { + break; + } + } + int month = 0; + while(true) { + int length = DAYS_IN_MONTH[month] + (month == 1 && isLeap(year) ? 1 : 0); + if(days < length) { + break; + } + days -= length; + month++; + } + + StringBuilder out = new StringBuilder(16); + pad(out, year, 4); + pad(out, month + 1, 2); + pad(out, (int)days + 1, 2); + out.append('T'); + pad(out, secondOfDay / 3600, 2); + pad(out, (secondOfDay / 60) % 60, 2); + pad(out, secondOfDay % 60, 2); + out.append('Z'); + return out.toString(); + } + + private static long floorDiv(long value, long divisor) { + long q = value / divisor; + if((value % divisor != 0) && ((value < 0) != (divisor < 0))) { + q--; + } + return q; + } + + private static boolean isLeap(int year) { + return (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; + } + + private static void pad(StringBuilder out, int value, int width) { + String text = String.valueOf(value); + for(int iter = text.length() ; iter < width ; iter++) { + out.append('0'); + } + out.append(text); + } +} diff --git a/vm/backend/src/com/codename1/backend/aws/Credentials.java b/vm/backend/src/com/codename1/backend/aws/Credentials.java new file mode 100644 index 00000000000..c330039af69 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/aws/Credentials.java @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.aws; + +import java.io.IOException; +import java.util.Map; + +import com.codename1.backend.Json; +import com.codename1.backend.Web; + +/** + * AWS credentials, and the ways a server actually obtains them. + * + * Deliberately in this order, which is the order the AWS SDKs use and the order + * that matters operationally: + * + * 1. The environment. This is what Lambda sets, what a local developer exports, + * and what a CI job injects. + * 2. The container credential endpoint. ECS and EKS publish a relative URI on + * 169.254.170.2 (or a full URI for EKS Pod Identity) that returns a temporary + * credential and refreshes it. This is how a task gets a ROLE rather than a + * long-lived key, which is the arrangement any reviewer will ask for. + * 3. The instance metadata service, IMDSv2 only. v1 is a plain GET that any + * process -- or any server-side request forgery -- can make; v2 requires a PUT + * to obtain a token first. Falling back to v1 would undo that, so this does + * not. + * + * Temporary credentials expire. {@link #isExpiring} says when to fetch again; + * {@link Session} does it. + */ +public final class Credentials { + private final String accessKeyId; + private final String secretKey; + private final String sessionToken; + private final long expiresAtMillis; + + public Credentials(String accessKeyId, String secretKey, String sessionToken) { + this(accessKeyId, secretKey, sessionToken, 0); + } + + public Credentials(String accessKeyId, String secretKey, String sessionToken, + long expiresAtMillis) { + this.accessKeyId = accessKeyId; + this.secretKey = secretKey; + this.sessionToken = sessionToken; + this.expiresAtMillis = expiresAtMillis; + } + + public String getAccessKeyId() { + return accessKeyId; + } + + public String getSecretKey() { + return secretKey; + } + + /** Null for a long-lived key pair; set for anything temporary. */ + public String getSessionToken() { + return sessionToken; + } + + /** 0 when these do not expire. */ + public long getExpiresAtMillis() { + return expiresAtMillis; + } + + /** + * True within `marginMillis` of expiry. A margin rather than the exact instant + * because a request signed just before expiry can still arrive just after it. + */ + public boolean isExpiring(long marginMillis) { + return expiresAtMillis > 0 + && System.currentTimeMillis() + marginMillis >= expiresAtMillis; + } + + /** + * The first source that answers, in the order documented on this class. + * Throws when none does, naming what was tried -- "no credentials" with no + * further detail is the least useful message a deployment can get. + */ + public static Credentials resolve() throws IOException { + Credentials fromEnvironment = fromEnvironment(); + if(fromEnvironment != null) { + return fromEnvironment; + } + Credentials fromContainer = fromContainer(); + if(fromContainer != null) { + return fromContainer; + } + Credentials fromInstance = fromInstanceMetadata(); + if(fromInstance != null) { + return fromInstance; + } + throw new IOException("No AWS credentials: AWS_ACCESS_KEY_ID is unset, " + + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI and " + + "AWS_CONTAINER_CREDENTIALS_FULL_URI are unset, and the instance " + + "metadata service did not answer"); + } + + /** AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN, or null. */ + public static Credentials fromEnvironment() { + String id = System.getenv("AWS_ACCESS_KEY_ID"); + String secret = System.getenv("AWS_SECRET_ACCESS_KEY"); + if(id == null || id.length() == 0 || secret == null || secret.length() == 0) { + return null; + } + String token = System.getenv("AWS_SESSION_TOKEN"); + return new Credentials(id, secret, + token == null || token.length() == 0 ? null : token); + } + + /** The ECS / EKS container credential endpoint, or null when not in one. */ + public static Credentials fromContainer() throws IOException { + String relative = System.getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); + String full = System.getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI"); + String url; + if(relative != null && relative.length() > 0) { + url = "http://169.254.170.2" + relative; + } else if(full != null && full.length() > 0) { + url = full; + } else { + return null; + } + java.util.List headers = new java.util.ArrayList(); + String tokenFile = System.getenv("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE"); + String token = System.getenv("AWS_CONTAINER_AUTHORIZATION_TOKEN"); + if(tokenFile != null && tokenFile.length() > 0) { + token = readFile(tokenFile); + } + if(token != null && token.length() > 0) { + headers.add("Authorization: " + token.trim()); + } + Web.Result result = Web.request("GET", url, headers, null); + if(!result.isSuccess()) { + throw new IOException("The container credential endpoint answered " + + result.getStatus()); + } + return fromJson(result.getBodyAsString()); + } + + /** + * IMDSv2. The PUT that obtains a token is the whole point: a v1 GET can be + * made by anything that can persuade this process to fetch a URL. + */ + public static Credentials fromInstanceMetadata() { + try { + java.util.List tokenHeaders = new java.util.ArrayList(); + tokenHeaders.add("X-aws-ec2-metadata-token-ttl-seconds: 300"); + Web.Result token = Web.request("PUT", + "http://169.254.169.254/latest/api/token", tokenHeaders, new byte[0]); + if(!token.isSuccess()) { + return null; + } + java.util.List headers = new java.util.ArrayList(); + headers.add("X-aws-ec2-metadata-token: " + token.getBodyAsString().trim()); + Web.Result roles = Web.request("GET", + "http://169.254.169.254/latest/meta-data/iam/security-credentials/", + headers, null); + if(!roles.isSuccess()) { + return null; + } + String role = roles.getBodyAsString().trim(); + int newline = role.indexOf('\n'); + if(newline > 0) { + role = role.substring(0, newline).trim(); + } + if(role.length() == 0) { + return null; + } + Web.Result body = Web.request("GET", + "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + role, + headers, null); + if(!body.isSuccess()) { + return null; + } + return fromJson(body.getBodyAsString()); + } catch (Exception err) { + // Not on EC2, or the link-local address is unreachable. That is not an + // error at this layer -- resolve() reports what it tried. + return null; + } + } + + /** + * The shape both endpoints return: AccessKeyId, SecretAccessKey, Token and + * Expiration. + */ + static Credentials fromJson(String json) throws IOException { + Map parsed = Json.parseObject(json); + String id = string(parsed, "AccessKeyId"); + String secret = string(parsed, "SecretAccessKey"); + if(id == null || secret == null) { + throw new IOException("The credential endpoint returned no key pair"); + } + String token = string(parsed, "Token"); + if(token == null) { + token = string(parsed, "SessionToken"); + } + return new Credentials(id, secret, token, expiryMillis(string(parsed, "Expiration"))); + } + + /** + * "2026-08-28T13:45:00Z" to epoch millis. Parsed by hand for the same reason + * {@link Clock} formats by hand: the translated runtime's date parsing is + * locale-aware and this format is not. + */ + static long expiryMillis(String iso) { + if(iso == null || iso.length() < 19) { + return 0; + } + try { + int year = Integer.parseInt(iso.substring(0, 4)); + int month = Integer.parseInt(iso.substring(5, 7)); + int day = Integer.parseInt(iso.substring(8, 10)); + int hour = Integer.parseInt(iso.substring(11, 13)); + int minute = Integer.parseInt(iso.substring(14, 16)); + int second = Integer.parseInt(iso.substring(17, 19)); + long days = 0; + for(int y = 1970 ; y < year ; y++) { + days += isLeap(y) ? 366 : 365; + } + int[] lengths = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + for(int m = 0 ; m < month - 1 ; m++) { + days += lengths[m] + (m == 1 && isLeap(year) ? 1 : 0); + } + days += day - 1; + return ((days * 24L + hour) * 60L + minute) * 60L * 1000L + second * 1000L; + } catch (Exception err) { + return 0; + } + } + + private static boolean isLeap(int year) { + return (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; + } + + private static String string(Map map, String key) { + Object value = map.get(key); + return value == null ? null : String.valueOf(value); + } + + private static String readFile(String path) throws IOException { + java.io.InputStream in = new java.io.FileInputStream(path); + try { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + byte[] chunk = new byte[4096]; + int n; + while((n = in.read(chunk)) > 0) { + out.write(chunk, 0, n); + } + return new String(out.toByteArray(), "UTF-8"); + } finally { + in.close(); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/aws/S3.java b/vm/backend/src/com/codename1/backend/aws/S3.java new file mode 100644 index 00000000000..1f0aaa8eb1a --- /dev/null +++ b/vm/backend/src/com/codename1/backend/aws/S3.java @@ -0,0 +1,508 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.aws; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.codename1.backend.Web; + +/** + * Amazon S3, and anything that speaks its API (MinIO, Cloudflare R2, Backblaze + * B2, Wasabi, Ceph) -- which is why the endpoint is configurable rather than + * assembled from a region alone. + * + * Two addressing styles exist and the choice is not cosmetic: virtual-hosted + * (`bucket.s3.region.amazonaws.com`) is what AWS requires for new buckets, and + * path-style (`endpoint/bucket/key`) is what a local MinIO or a bucket whose name + * is not DNS-safe needs. Both are supported because a backend is usually + * developed against the second and deployed against the first. + * + * The presigned URL is the method to reach for from a mobile app: it lets the + * device upload or download directly and keeps the object bytes out of the + * server, which is most of the reason to use object storage from an app. + */ +public final class S3 { + /** + * How long before expiry to fetch again. Long enough that a request signed now + * is still valid when it arrives, short enough not to refresh constantly. + */ + private static final long CREDENTIAL_REFRESH_MARGIN = 5 * 60 * 1000L; + + private Credentials credentials; + private final String region; + private final String endpoint; + private final boolean pathStyle; + private final boolean secure; + private final boolean fromEnvironment; + + private S3(Credentials credentials, String region, String endpoint, boolean pathStyle, + boolean secure, boolean fromEnvironment) { + this.credentials = credentials; + this.region = region; + this.endpoint = endpoint; + this.pathStyle = pathStyle; + this.secure = secure; + this.fromEnvironment = fromEnvironment; + } + + /** + * The credentials to sign with, resolved again when a temporary one is near expiry. + * + * The role credentials ECS, EKS, EC2 and Lambda hand out live for minutes to + * hours, so a server that captured one at startup would spend the rest of its + * life signing with a credential the service has already forgotten. Only the + * environment-resolved case can be refreshed: a credential passed to + * {@link #forEndpoint} is the caller's, and there is no provider to ask again. + * + * Two request threads can resolve at once here and one of the two answers is + * dropped. That is harmless -- both are valid credentials, and the cost of the + * duplicate call is one metadata round trip an hour. + */ + private Credentials credentials() throws IOException { + if(fromEnvironment && credentials.isExpiring(CREDENTIAL_REFRESH_MARGIN)) { + credentials = Credentials.resolve(); + } + return credentials; + } + + /** + * AWS S3 in one region, with credentials resolved the usual way. + * + * The region is taken from AWS_REGION when not given, because that is what + * Lambda and ECS set and hard-coding it is how a service ends up deployable in + * exactly one place. + */ + public static S3 forRegion(String region) throws IOException { + String resolved = region != null && region.length() > 0 + ? region : System.getenv("AWS_REGION"); + if(resolved == null || resolved.length() == 0) { + resolved = System.getenv("AWS_DEFAULT_REGION"); + } + if(resolved == null || resolved.length() == 0) { + throw new IOException("No AWS region: pass one, or set AWS_REGION"); + } + return new S3(Credentials.resolve(), resolved, + "s3." + resolved + ".amazonaws.com", false, true, true); + } + + /** + * An S3-compatible endpoint -- MinIO, R2, Ceph -- addressed path-style. + * + * `endpoint` is a host with an optional port and an optional scheme: + * "minio.internal", "localhost:9000", "http://localhost:9000". TLS is assumed + * unless the endpoint says http:// -- a default of "encrypted" is the one + * that fails safely. + */ + public static S3 forEndpoint(Credentials credentials, String region, String endpoint) { + String host = endpoint == null ? "" : endpoint; + boolean useTls = true; + if(host.startsWith("http://")) { + useTls = false; + host = host.substring("http://".length()); + } else if(host.startsWith("https://")) { + host = host.substring("https://".length()); + } + while(host.endsWith("/")) { + host = host.substring(0, host.length() - 1); + } + return new S3(credentials, region == null ? "us-east-1" : region, host, true, useTls, + false); + } + + /** + * Creates a bucket, and says nothing when it already exists. + * + * Usually infrastructure's job rather than the application's, but a first run + * against a fresh MinIO or a test fixture needs it, and the alternative is a + * shell script that speaks a protocol this class already speaks. + */ + public void createBucket(String bucket) throws IOException { + Web.Result result = send("PUT", bucket, "", null, null, createBucketBody()); + if(result.isSuccess()) { + return; + } + List code = elements(result.getBodyAsString(), "Code"); + String reason = code.isEmpty() ? "" : String.valueOf(code.get(0)); + // Only "owned by you" is the idempotent case. Bucket names are global on AWS, + // so "already exists" means somebody else has it: returning normally there + // would report success for a bucket the caller does not have and cannot use, + // and every later call would fail on authorization instead of here. + if("BucketAlreadyOwnedByYou".equals(reason)) { + return; + } + requireSuccess(result, "CREATE BUCKET", bucket, ""); + } + + /** Uploads an object. Returns its ETag, which is the server's receipt. */ + public String putObject(String bucket, String key, byte[] content, String contentType) + throws IOException { + Map headers = new LinkedHashMap(); + headers.put("content-type", contentType == null + ? "application/octet-stream" : contentType); + Web.Result result = send("PUT", bucket, key, null, headers, + content == null ? new byte[0] : content); + requireSuccess(result, "PUT", bucket, key); + String etag = result.getHeader("etag"); + return etag == null ? "" : etag.replace("\"", ""); + } + + /** Downloads an object. Throws when it is missing, rather than returning null. */ + public byte[] getObject(String bucket, String key) throws IOException { + Web.Result result = send("GET", bucket, key, null, null, null); + requireSuccess(result, "GET", bucket, key); + return result.getBody(); + } + + /** The object's metadata, or null when it does not exist. */ + public ObjectInfo headObject(String bucket, String key) throws IOException { + Web.Result result = send("HEAD", bucket, key, null, null, null); + if(result.getStatus() == 404) { + return null; + } + requireSuccess(result, "HEAD", bucket, key); + ObjectInfo info = new ObjectInfo(); + info.key = key; + info.size = parseLong(result.getHeader("content-length")); + info.contentType = result.getHeader("content-type"); + String etag = result.getHeader("etag"); + info.etag = etag == null ? null : etag.replace("\"", ""); + info.lastModified = result.getHeader("last-modified"); + return info; + } + + public void deleteObject(String bucket, String key) throws IOException { + Web.Result result = send("DELETE", bucket, key, null, null, null); + // S3 answers 204 for a delete, and also for a key that was not there. + if(result.getStatus() != 204 && result.getStatus() != 200) { + requireSuccess(result, "DELETE", bucket, key); + } + } + + /** + * Lists up to `max` objects under a prefix. + * + * ListObjectsV2, and paginated: S3 caps a page at 1000 keys whatever you ask + * for, and a caller that ignores the continuation token silently sees only the + * first page. This follows the token until the listing is complete or `max` is + * reached. + */ + public List listObjects(String bucket, String prefix, int max) throws IOException { + List keys = new ArrayList(); + String token = null; + while(true) { + Map query = new LinkedHashMap(); + query.put("list-type", "2"); + if(prefix != null && prefix.length() > 0) { + query.put("prefix", prefix); + } + if(token != null) { + query.put("continuation-token", token); + } + Web.Result result = send("GET", bucket, "", query, null, null); + requireSuccess(result, "LIST", bucket, prefix == null ? "" : prefix); + String body = result.getBodyAsString(); + List page = elements(body, "Key"); + for(int iter = 0 ; iter < page.size() ; iter++) { + keys.add(page.get(iter)); + if(max > 0 && keys.size() >= max) { + return keys; + } + } + List next = elements(body, "NextContinuationToken"); + if(next.isEmpty()) { + return keys; + } + token = (String)next.get(0); + } + } + + /** + * A URL that downloads the object without any credentials, for `seconds`. + * + * Nothing is sent here: a presigned URL is a computation, so this costs no + * round trip and can be handed straight to a client. + */ + public String presignGet(String bucket, String key, int seconds) throws IOException { + return Aws.presign(credentials(), region, "s3", "GET", hostFor(bucket), + pathFor(bucket, key), null, seconds, null, secure); + } + + /** The upload counterpart: a URL a client can PUT to, for `seconds`. */ + public String presignPut(String bucket, String key, int seconds) throws IOException { + return Aws.presign(credentials(), region, "s3", "PUT", hostFor(bucket), + pathFor(bucket, key), null, seconds, null, secure); + } + + /** What {@link #headObject} reports. */ + public static final class ObjectInfo { + String key; + long size; + String contentType; + String etag; + String lastModified; + + public String getKey() { + return key; + } + + public long getSize() { + return size; + } + + public String getContentType() { + return contentType; + } + + public String getEtag() { + return etag; + } + + /** The raw HTTP date the service sent, not a parsed one. */ + public String getLastModified() { + return lastModified; + } + } + + /** + * CreateBucket's body names the region, except in us-east-1 where it must not. + * + * S3 reads an empty CreateBucket as a request for us-east-1, so a bucket asked + * for anywhere else comes back as IllegalLocationConstraintException unless the + * body says where -- and us-east-1 rejects the body that says so. The + * S3-compatible endpoints take the empty body, which is what pathStyle + * distinguishes: forEndpoint addresses those path-style, forRegion does not. + */ + private byte[] createBucketBody() throws IOException { + if(pathStyle || "us-east-1".equals(region)) { + return new byte[0]; + } + return ("" + + "" + region + "" + + "").getBytes("UTF-8"); + } + + private Web.Result send(String method, String bucket, String key, Map query, + Map headers, byte[] body) throws IOException { + return Aws.send(credentials(), region, "s3", method, hostFor(bucket), + pathFor(bucket, key), query, headers, body, null, secure); + } + + /** + * Refuses a bucket name that could change the host this request goes to. + * + * Virtual-hosted addressing puts the name in front of the endpoint and the + * result is concatenated straight after "https://", so a name carrying a + * slash -- "attacker.example/ignored" -- makes the authority the attacker's + * host and the rest a path. The request then carries the signed access key + * identifier and session token there. A caller that derives the name from + * tenant or request input is the case this exists for; one that hard-codes + * it loses nothing, because a name that fails this could not have resolved + * as a hostname anyway. + * + * These are S3's own rules for a DNS-compatible name: 3 to 63 characters of + * lowercase letter, digit, dot or hyphen, beginning and ending with a letter + * or digit, and no two dots in a row. + */ + private static void requireDnsBucket(String bucket) { + int length = bucket == null ? 0 : bucket.length(); + boolean ok = length >= 3 && length <= 63; + for(int iter = 0 ; ok && iter < length ; iter++) { + char c = bucket.charAt(iter); + boolean alnum = (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); + if(!alnum && c != '.' && c != '-') { + ok = false; + } else if((iter == 0 || iter == length - 1) && !alnum) { + ok = false; + } else if(c == '.' && iter > 0 && bucket.charAt(iter - 1) == '.') { + ok = false; + } + } + if(!ok) { + throw new IllegalArgumentException("Not a DNS-compatible S3 bucket " + + "name, so it cannot be addressed virtual-hosted: " + bucket); + } + } + + /** + * Refuses a bucket name that could change the PATH this request addresses. + * + * Deliberately narrower than the DNS rules above. Path style is what a bucket + * that cannot satisfy those rules uses -- the legacy us-east-1 names with + * uppercase and underscores are exactly that -- so applying them here would + * refuse the buckets this addressing mode exists to serve. What matters when + * the name goes into the path is only that it stays one segment. + */ + private static void requirePathSafeBucket(String bucket) { + if(bucket == null || bucket.length() == 0 || bucket.indexOf('/') >= 0 + || bucket.indexOf('\\') >= 0 || bucket.indexOf("..") >= 0) { + throw new IllegalArgumentException("An S3 bucket name cannot contain a " + + "path separator or \"..\": " + bucket); + } + } + + /** + * Whether THIS bucket has to be addressed path style. + * + * A dotted name cannot go in front of the endpoint over TLS. The wildcard + * certificate for `*.s3..amazonaws.com` matches exactly one label, so + * "photos.example" would need it to match two and every request -- and every + * presigned URL handed to a client -- fails hostname verification. The name + * is perfectly legal; it is the addressing that cannot carry it, so the + * request takes the path form instead of failing. + */ + private boolean usesPathStyle(String bucket) { + return pathStyle || (secure && bucket != null && bucket.indexOf('.') >= 0); + } + + private String hostFor(String bucket) { + if(!usesPathStyle(bucket)) { + requireDnsBucket(bucket); + return bucket + "." + endpoint; + } + return endpoint; + } + + private String pathFor(String bucket, String key) { + String suffix = key == null ? "" : key; + if(!usesPathStyle(bucket)) { + return "/" + suffix; + } + // Checked here rather than only in send(): presign() calls hostFor and + // pathFor directly, so a check on the send path alone would leave the two + // presigning entry points unguarded. + requirePathSafeBucket(bucket); + return "/" + bucket + "/" + suffix; + } + + /** + * S3 reports failures as an XML body with a Code and a Message, and the status + * alone ("403") does not say whether the key, the bucket, the signature or the + * clock is at fault. Both go into the exception. + */ + private static void requireSuccess(Web.Result result, String operation, String bucket, + String key) throws IOException { + if(result.isSuccess()) { + return; + } + String body = result.getBodyAsString(); + List code = elements(body, "Code"); + List message = elements(body, "Message"); + throw new IOException("S3 " + operation + " s3://" + bucket + "/" + key + + " failed with " + result.getStatus() + + (code.isEmpty() ? "" : " " + code.get(0)) + + (message.isEmpty() ? "" : ": " + message.get(0))); + } + + /** + * The text of every <name> element, in order. + * + * A deliberate non-parser: S3's list and error responses are flat, the element + * names wanted are known, and a real XML parser is a dependency this runtime + * does not have. It decodes the five predefined entities, which is what S3 + * escapes in a key. + */ + static List elements(String xml, String name) { + List out = new ArrayList(); + if(xml == null) { + return out; + } + String open = "<" + name + ">"; + String close = ""; + int at = 0; + while(true) { + int start = xml.indexOf(open, at); + if(start < 0) { + return out; + } + int end = xml.indexOf(close, start + open.length()); + if(end < 0) { + return out; + } + out.add(unescape(xml.substring(start + open.length(), end))); + at = end + close.length(); + } + } + + static String unescape(String value) { + if(value.indexOf('&') < 0) { + return value; + } + StringBuilder out = new StringBuilder(value.length()); + int at = 0; + while(at < value.length()) { + char c = value.charAt(at); + if(c != '&') { + out.append(c); + at++; + continue; + } + int semi = value.indexOf(';', at); + if(semi < 0) { + out.append(c); + at++; + continue; + } + String entity = value.substring(at + 1, semi); + if("amp".equals(entity)) { + out.append('&'); + } else if("lt".equals(entity)) { + out.append('<'); + } else if("gt".equals(entity)) { + out.append('>'); + } else if("quot".equals(entity)) { + out.append('"'); + } else if("apos".equals(entity)) { + out.append('\''); + } else if(entity.length() > 1 && entity.charAt(0) == '#') { + try { + int code = entity.charAt(1) == 'x' || entity.charAt(1) == 'X' + ? Integer.parseInt(entity.substring(2), 16) + : Integer.parseInt(entity.substring(1)); + out.append((char)code); + } catch (NumberFormatException err) { + out.append('&').append(entity).append(';'); + } + } else { + out.append('&').append(entity).append(';'); + } + at = semi + 1; + } + return out.toString(); + } + + private static long parseLong(String value) { + if(value == null) { + return -1; + } + try { + return Long.parseLong(value.trim()); + } catch (NumberFormatException err) { + return -1; + } + } +} diff --git a/vm/backend/src/com/codename1/backend/sql/MySql.java b/vm/backend/src/com/codename1/backend/sql/MySql.java new file mode 100644 index 00000000000..d4353ffad30 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/sql/MySql.java @@ -0,0 +1,1008 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.sql; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.codename1.backend.Crypto; +import com.codename1.backend.Tcp; + +/** + * A MySQL client speaking the client/server protocol directly, for the same + * reason as {@link Postgres}: a translated server binary has no JDBC. + * + * Everything goes through prepared statements (COM_STMT_PREPARE / EXECUTE) and + * therefore through MySQL's BINARY result format. That is more code than sending + * COM_QUERY text, and it is the only way to bind a parameter -- a text-protocol + * client has to build SQL by concatenation, and this API refuses to offer that. + * + * Authentication covers what is actually deployed: caching_sha2_password (the + * MySQL 8 default) and mysql_native_password (5.7 and MariaDB). The caching_sha2 + * FULL exchange -- which the server demands the first time a password is used, + * before its cache is warm -- sends the password to the server, so this client + * does it only on a TLS connection and says so rather than falling back to the + * RSA-wrapped variant, which would be a second cryptographic path to get wrong. + */ +public final class MySql { + /** Capability bits, from the protocol's CLIENT_* set. */ + private static final int CLIENT_LONG_PASSWORD = 0x00000001; + private static final int CLIENT_LONG_FLAG = 0x00000004; + private static final int CLIENT_CONNECT_WITH_DB = 0x00000008; + private static final int CLIENT_LOCAL_FILES = 0x00000080; + private static final int CLIENT_PROTOCOL_41 = 0x00000200; + private static final int CLIENT_SSL = 0x00000800; + private static final int CLIENT_TRANSACTIONS = 0x00002000; + private static final int CLIENT_SECURE_CONNECTION = 0x00008000; + private static final int CLIENT_PLUGIN_AUTH = 0x00080000; + private static final int CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA = 0x00200000; + + private static final int OK_PACKET = 0x00; + private static final int EOF_PACKET = 0xfe; + private static final int ERROR_PACKET = 0xff; + + private final Wire wire; + private int sequence; + private long lastInsertId; + private boolean closed; + + private MySql(Wire wire) { + this.wire = wire; + } + + /** + * Connects, optionally upgrades to TLS, and authenticates. + * + * `sslMode` is "require", "prefer" or "disable", as in {@link Postgres}. + */ + public static MySql connect(String host, int port, String database, String user, + String password, String sslMode, String caFile, int timeoutMillis) throws IOException { + Tcp connection = Tcp.connect(host, port <= 0 ? 3306 : port, timeoutMillis); + try { + MySql session = new MySql(new Wire(connection)); + session.handshake(host, database, user, password, sslMode, caFile); + return session; + } catch (IOException err) { + connection.close(); + throw err; + } + } + + private void handshake(String host, String database, String user, String password, + String sslMode, String caFile) throws IOException { + Packet greeting = readPacket(); + Reader reader = new Reader(greeting.body); + int protocol = reader.u8(); + if(protocol == ERROR_PACKET) { + throw errorFrom(greeting, null); + } + if(protocol != 10) { + throw new IOException("Unsupported MySQL handshake protocol " + protocol); + } + reader.cString(); // server version + reader.skip(4); // connection id + byte[] scrambleFirst = reader.bytes(8); + reader.skip(1); // filler + int serverCapabilities = reader.u16(); + byte[] scramble = scrambleFirst; + String plugin = "mysql_native_password"; + if(reader.remaining() > 0) { + reader.skip(1); // character set + reader.skip(2); // status flags + serverCapabilities |= reader.u16() << 16; + int scrambleLength = reader.u8(); + reader.skip(10); // reserved + // The documented length is the total including the first 8 bytes and a + // trailing NUL, and servers disagree about the NUL -- so take what is + // there rather than what is claimed. + int secondLength = scrambleLength > 8 ? scrambleLength - 8 : 12; + if(secondLength > reader.remaining()) { + secondLength = reader.remaining(); + } + byte[] scrambleSecond = reader.bytes(secondLength); + scramble = trimTrailingNul(concat(scrambleFirst, scrambleSecond)); + if(reader.remaining() > 0) { + plugin = reader.cString(); + } + } + + boolean useTls = !"disable".equals(sslMode); + if(useTls && (serverCapabilities & CLIENT_SSL) == 0) { + if("require".equals(sslMode)) { + throw new IOException("The MySQL server at " + host + + " does not offer TLS and sslmode=require"); + } + useTls = false; + } + + // Deliberately NOT CLIENT_FOUND_ROWS. With it MySQL reports the rows an + // UPDATE MATCHED rather than the rows it changed, so an update that found + // its row and altered nothing answers 1 where Database.execute documents + // "the number of rows changed" and where SQLite and Postgres both answer + // 0. Code that reads 0 as "no such row" would have been told it succeeded. + int capabilities = CLIENT_LONG_PASSWORD | CLIENT_LONG_FLAG + | CLIENT_PROTOCOL_41 | CLIENT_TRANSACTIONS | CLIENT_SECURE_CONNECTION + | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA; + if(database != null && database.length() > 0) { + capabilities |= CLIENT_CONNECT_WITH_DB; + } + if(useTls) { + capabilities |= CLIENT_SSL; + // The SSLRequest packet is the first 32 bytes of the login packet and + // nothing else: the credentials must not cross in the clear, which is + // the entire point of sending it separately. + ByteArrayOutputStream request = new ByteArrayOutputStream(); + writeIntLE(request, capabilities); + writeIntLE(request, 0x01000000); // max packet size + request.write(45); // utf8mb4_general_ci + for(int iter = 0 ; iter < 23 ; iter++) { + request.write(0); + } + sendPacket(request.toByteArray()); + try { + wire.getConnection().startTls(host, caFile); + } catch (IOException err) { + // The SSLRequest packet has already gone out, so this connection + // cannot go back to plaintext -- the server is waiting for a + // handshake. Saying what to do beats a bare PKIX stack trace, + // which is what a self-signed development server produces. + throw new IOException("TLS to " + host + " could not be verified (" + + err.getMessage() + "). Point sslrootcert at the server's " + + "CA, or set sslmode=disable to connect in the clear " + + "deliberately."); + } + } + // LOCAL INFILE lets a server ask the client for a file by path. Nothing + // here needs it, and leaving it enabled turns a compromised or hostile + // server into a file read on this host. + capabilities &= ~CLIENT_LOCAL_FILES; + + byte[] authResponse = authResponse(plugin, password, scramble); + ByteArrayOutputStream login = new ByteArrayOutputStream(); + writeIntLE(login, capabilities); + writeIntLE(login, 0x01000000); + login.write(45); + for(int iter = 0 ; iter < 23 ; iter++) { + login.write(0); + } + writeCString(login, user); + writeLengthEncoded(login, authResponse); + if((capabilities & CLIENT_CONNECT_WITH_DB) != 0) { + writeCString(login, database); + } + writeCString(login, plugin); + sendPacket(login.toByteArray()); + + finishAuthentication(password, scramble, useTls); + } + + /** + * Drives the post-login exchange: an OK ends it, an AuthSwitchRequest changes + * plugin, and caching_sha2_password may ask for the full exchange. + */ + private void finishAuthentication(String password, byte[] scramble, boolean secure) + throws IOException { + while(true) { + Packet packet = readPacket(); + int head = packet.body[0] & 0xff; + if(head == OK_PACKET) { + return; + } + if(head == ERROR_PACKET) { + throw errorFrom(packet, null); + } + if(head == EOF_PACKET) { + // AuthSwitchRequest: plugin name, then a fresh scramble. + Reader reader = new Reader(packet.body); + reader.skip(1); + String plugin = reader.cString(); + byte[] fresh = trimTrailingNul(reader.rest()); + sendPacket(authResponse(plugin, password, fresh)); + scramble = fresh; + continue; + } + if(head == 0x01) { + // AuthMoreData. For caching_sha2_password 0x03 means the server's + // cache already had this password and 0x04 means it did not. + int status = packet.body.length > 1 ? packet.body[1] & 0xff : 0; + if(status == 3) { + continue; // fast path accepted; an OK follows + } + if(status == 4) { + if(!secure) { + throw new IOException("This MySQL server needs the full " + + "caching_sha2_password exchange, which sends the " + + "password; connect with sslmode=require (or run " + + "ALTER USER ... IDENTIFIED WITH mysql_native_password)"); + } + byte[] clear = Wire.utf8(password == null ? "" : password); + byte[] terminated = new byte[clear.length + 1]; + System.arraycopy(clear, 0, terminated, 0, clear.length); + sendPacket(terminated); + continue; + } + throw new IOException("Unexpected MySQL auth continuation " + status); + } + throw new IOException("Unexpected MySQL authentication packet 0x" + + Integer.toHexString(head)); + } + } + + private static byte[] authResponse(String plugin, String password, byte[] scramble) + throws IOException { + byte[] secret = Wire.utf8(password == null ? "" : password); + if(secret.length == 0) { + return new byte[0]; + } + if("caching_sha2_password".equals(plugin)) { + // XOR(SHA256(password), SHA256(SHA256(SHA256(password)) + scramble)) + byte[] first = Crypto.sha256(secret); + byte[] second = Crypto.sha256(first); + byte[] third = Crypto.sha256(concat(second, scramble)); + return xor(first, third); + } + // NOT mysql_old_password. It was accepted here and answered with the + // mysql_native_password scramble, which is a different algorithm entirely -- + // the pre-4.1 one -- so such an account was always rejected by the server + // while the message below already said this client does not speak it. The + // plugin is removed in MySQL 8.0 and its hash is broken by design, so it + // falls through to that message rather than being implemented. + if("mysql_native_password".equals(plugin)) { + // XOR(SHA1(password), SHA1(scramble + SHA1(SHA1(password)))) + byte[] first = Crypto.sha1(secret); + byte[] second = Crypto.sha1(first); + byte[] third = Crypto.sha1(concat(scramble, second)); + return xor(first, third); + } + throw new IOException("Unsupported MySQL authentication plugin '" + plugin + + "'; this client speaks caching_sha2_password and mysql_native_password"); + } + + // ---------------- queries ---------------- + + public int execute(String sql, Object[] params) throws IOException { + return (int)runPrepared(sql, params, null); + } + + public List query(String sql, Object[] params) throws IOException { + List rows = new ArrayList(); + runPrepared(sql, params, rows); + return rows; + } + + /** + * Runs a statement through COM_QUERY rather than the prepared-statement + * protocol, and takes no parameters. + * + * This exists for exactly one reason: MySQL refuses to prepare its + * transaction-control statements ("This command is not supported in the + * prepared statement protocol yet"), so BEGIN, COMMIT and ROLLBACK cannot go + * through {@link #execute}. It is private, and the three callers below pass + * constants -- a text-protocol entry point taking a caller's string is the + * concatenation hole this client exists to avoid. + */ + private void command(String sql) throws IOException { + checkOpen(); + sequence = 0; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write(0x03); // COM_QUERY + byte[] text = Wire.utf8(sql); + out.write(text, 0, text.length); + sendPacket(out.toByteArray()); + + Packet packet = readPacket(); + int head = packet.body[0] & 0xff; + if(head == ERROR_PACKET) { + throw errorFrom(packet, sql); + } + if(head == OK_PACKET || head == EOF_PACKET) { + return; + } + // A result set. Nothing here wants the rows, but they have to be drained + // or the next statement reads them as its own answer. + Reader header = new Reader(packet.body); + int columns = (int)header.lengthEncoded(); + for(int iter = 0 ; iter < columns ; iter++) { + readPacket(); + } + readPacket(); // EOF closing the definitions + while(true) { + Packet row = readPacket(); + int marker = row.body[0] & 0xff; + if(marker == ERROR_PACKET) { + throw errorFrom(row, sql); + } + if(marker == EOF_PACKET && row.body.length < 9) { + return; + } + } + } + + /** Opens a transaction. See {@link #command} for why this is not `execute`. */ + public void begin() throws IOException { + command("BEGIN"); + } + + public void commit() throws IOException { + command("COMMIT"); + } + + public void rollback() throws IOException { + command("ROLLBACK"); + } + + public long lastInsertId() { + return lastInsertId; + } + + /** + * Prepares, executes and closes one statement. Returns the affected-row count + * and, when `rows` is not null, appends the decoded result set to it. + * + * The statement is closed rather than cached: a cache keyed by SQL text is + * where a pooled connection starts leaking server-side handles, and preparing + * costs one round trip. + */ + private long runPrepared(String sql, Object[] params, List rows) throws IOException { + checkOpen(); + sequence = 0; + ByteArrayOutputStream prepare = new ByteArrayOutputStream(); + prepare.write(0x16); // COM_STMT_PREPARE + byte[] text = Wire.utf8(sql); + prepare.write(text, 0, text.length); + sendPacket(prepare.toByteArray()); + + Packet response = readPacket(); + if((response.body[0] & 0xff) == ERROR_PACKET) { + throw errorFrom(response, sql); + } + Reader reader = new Reader(response.body); + reader.skip(1); + int statementId = reader.i32(); + int columnCount = reader.u16(); + int parameterCount = reader.u16(); + // Definitions for the parameters and then the columns, each list closed by + // an EOF packet. They are read and discarded for parameters -- the values + // are typed by this client, not by the server's guess. + if(parameterCount > 0) { + for(int iter = 0 ; iter < parameterCount ; iter++) { + readPacket(); + } + readPacket(); // EOF + } + Column[] columns = new Column[columnCount]; + if(columnCount > 0) { + for(int iter = 0 ; iter < columnCount ; iter++) { + columns[iter] = parseColumn(readPacket().body); + } + readPacket(); // EOF + } + + try { + return executePrepared(statementId, params, columns, rows, sql); + } finally { + sequence = 0; + ByteArrayOutputStream close = new ByteArrayOutputStream(); + close.write(0x19); // COM_STMT_CLOSE, which the server does not answer + writeIntLE(close, statementId); + try { + sendPacket(close.toByteArray()); + } catch (IOException ignored) { + // the connection is already broken; the caller sees the real error + } + } + } + + private long executePrepared(int statementId, Object[] params, Column[] columns, + List rows, String sql) throws IOException { + sequence = 0; + int count = params == null ? 0 : params.length; + ByteArrayOutputStream execute = new ByteArrayOutputStream(); + execute.write(0x17); // COM_STMT_EXECUTE + writeIntLE(execute, statementId); + execute.write(0); // no cursor + writeIntLE(execute, 1); // iteration count, always 1 + if(count > 0) { + byte[] nulls = new byte[(count + 7) / 8]; + for(int iter = 0 ; iter < count ; iter++) { + if(params[iter] == null) { + nulls[iter / 8] |= (byte)(1 << (iter % 8)); + } + } + execute.write(nulls, 0, nulls.length); + execute.write(1); // the types that follow are new + for(int iter = 0 ; iter < count ; iter++) { + int type = typeOf(params[iter]); + execute.write(type); + execute.write(0); // unsigned flag + } + for(int iter = 0 ; iter < count ; iter++) { + writeBinaryValue(execute, params[iter]); + } + } + sendPacket(execute.toByteArray()); + + Packet first = readPacket(); + int head = first.body[0] & 0xff; + if(head == ERROR_PACKET) { + throw errorFrom(first, sql); + } + if(head == OK_PACKET && columns.length == 0) { + Reader reader = new Reader(first.body); + reader.skip(1); + long affected = reader.lengthEncoded(); + long generated = reader.lengthEncoded(); + // Only when the statement actually generated one. Every successful + // command answers with an OK packet, and an UPDATE or a DDL reports + // zero here -- so assigning unconditionally let the next statement + // after an insert wipe the id, and lastInsertId() is documented as + // the MOST RECENT INSERT's. The SQLite and Java SE arms both keep + // the last generated key, and the arms must not disagree. + if(generated != 0) { + lastInsertId = generated; + } + return affected; + } + // A result set: a column count, the definitions again, then binary rows. + Reader header = new Reader(first.body); + int resultColumns = (int)header.lengthEncoded(); + Column[] resultDefinitions = new Column[resultColumns]; + for(int iter = 0 ; iter < resultColumns ; iter++) { + resultDefinitions[iter] = parseColumn(readPacket().body); + } + readPacket(); // EOF ending the definitions + while(true) { + Packet packet = readPacket(); + int marker = packet.body[0] & 0xff; + if(marker == ERROR_PACKET) { + throw errorFrom(packet, sql); + } + // An EOF packet is under 9 bytes; a row whose first byte is 0xfe is + // longer, which is how the two are told apart. + if(marker == EOF_PACKET && packet.body.length < 9) { + return 0; + } + if(rows != null) { + rows.add(decodeBinaryRow(packet.body, resultDefinitions)); + } + } + } + + /** + * A binary row: a 0x00 marker, a null bitmap offset by two bits, then each + * non-null value in its column's binary form. + */ + private static Map decodeBinaryRow(byte[] body, Column[] columns) throws IOException { + Reader reader = new Reader(body); + reader.skip(1); + byte[] nulls = reader.bytes((columns.length + 9) / 8); + Map row = new LinkedHashMap(); + for(int iter = 0 ; iter < columns.length ; iter++) { + int bit = iter + 2; + boolean isNull = (nulls[bit / 8] & (1 << (bit % 8))) != 0; + row.put(columns[iter].name, isNull ? null : readBinaryValue(reader, columns[iter])); + } + return row; + } + + /** + * Decoded to the same Java types the SQLite and PostgreSQL paths produce. + * Everything textual is a String unless its column is binary (character set + * 63), which is what separates a BLOB from a TEXT on this wire. + */ + private static Object readBinaryValue(Reader reader, Column column) throws IOException { + switch(column.type) { + case 0x01: // TINY + return Long.valueOf(column.unsigned ? reader.u8() : (byte)reader.u8()); + case 0x02: // SHORT + case 0x0d: // YEAR + return Long.valueOf(column.unsigned ? reader.u16() : (short)reader.u16()); + case 0x03: // LONG + case 0x09: // INT24 + return Long.valueOf(column.unsigned ? (reader.i32() & 0xffffffffL) : reader.i32()); + case 0x08: { // LONGLONG + // BIGINT UNSIGNED above Long.MAX_VALUE has no long that holds it: + // the high bit is a sign bit to Java, so the value comes back + // NEGATIVE -- an id or a counter arriving as a different number + // than the row holds, in the query result and in the JSON built + // from it. Such a value keeps its exact unsigned decimal as text, + // the same answer DECIMAL gets above and numeric gets on the + // PostgreSQL side: a type the API cannot hold is not rounded or + // wrapped into one that fits. Everything that DOES fit stays a + // Long, which is every signed BIGINT and every unsigned one below + // the boundary. + long value = reader.i64(); + if(column.unsigned && value < 0) { + return unsignedText(value); + } + return Long.valueOf(value); + } + case 0x04: // FLOAT + return Double.valueOf(Float.intBitsToFloat(reader.i32())); + case 0x05: // DOUBLE + return Double.valueOf(Double.longBitsToDouble(reader.i64())); + case 0x0a: // DATE + case 0x0c: // DATETIME + case 0x07: // TIMESTAMP + return reader.temporal(); + case 0x0b: // TIME + return reader.time(); + case 0x00: // DECIMAL + case 0xf6: { // NEWDECIMAL + // Sent as its decimal TEXT even in the binary protocol, and + // flagged character set 63 like every other numeric column -- + // so the binary fallback below would hand a money column back + // as a byte[], which JSON then base64s. Kept as the exact text + // rather than parsed to a double: DECIMAL(65,30) is why the + // column type was chosen, and a double cannot hold it. + byte[] digits = reader.lengthEncodedBytes(); + return digits == null ? null : Wire.fromUtf8(digits); + } + default: { + byte[] data = reader.lengthEncodedBytes(); + if(data == null) { + return null; + } + return column.binary ? (Object)data : (Object)Wire.fromUtf8(data); + } + } + } + + /** + * The exact decimal for a 64-bit value read as unsigned. Long.toString would + * print the negative wrap, and there is no unsigned formatter to call here, + * so it is divided out by hand: the top bit is worth 2^63, and the rest is + * an ordinary positive long. + */ + private static String unsignedText(long value) { + long quotient = (value >>> 1) / 5; // value / 10, unsigned + long remainder = value - quotient * 10; + if(remainder > 9) { // the halving can be one low + quotient += remainder / 10; + remainder %= 10; + } + return Long.toString(quotient) + (char)('0' + remainder); + } + + private static int typeOf(Object value) { + if(value == null) { + return 0x06; // NULL + } + if(value instanceof Integer || value instanceof Long || value instanceof Short + || value instanceof Byte || value instanceof Boolean) { + return 0x08; // LONGLONG, so one encoder covers every integer width + } + if(value instanceof Double || value instanceof Float) { + return 0x05; // DOUBLE + } + if(value instanceof byte[]) { + return 0xfc; // BLOB + } + return 0xfd; // VAR_STRING + } + + private static void writeBinaryValue(ByteArrayOutputStream out, Object value) { + if(value == null) { + return; // carried by the null bitmap, with no bytes on the wire + } + if(value instanceof Boolean) { + writeLongLE(out, ((Boolean)value).booleanValue() ? 1 : 0); + return; + } + if(value instanceof Integer || value instanceof Long || value instanceof Short + || value instanceof Byte) { + writeLongLE(out, ((Number)value).longValue()); + return; + } + if(value instanceof Double || value instanceof Float) { + writeLongLE(out, Double.doubleToLongBits(((Number)value).doubleValue())); + return; + } + if(value instanceof byte[]) { + writeLengthEncoded(out, (byte[])value); + return; + } + writeLengthEncoded(out, Wire.utf8(String.valueOf(value))); + } + + private static Column parseColumn(byte[] body) throws IOException { + Reader reader = new Reader(body); + reader.lengthEncodedBytes(); // catalog + reader.lengthEncodedBytes(); // schema + reader.lengthEncodedBytes(); // table + reader.lengthEncodedBytes(); // original table + byte[] name = reader.lengthEncodedBytes(); + reader.lengthEncodedBytes(); // original name + reader.lengthEncoded(); // length of the fixed fields + Column column = new Column(); + column.name = Wire.fromUtf8(name); + column.binary = reader.u16() == 63; // character set 63 is "binary" + reader.skip(4); // column length + column.type = reader.u8(); + // The flags follow the type, and 0x0020 is UNSIGNED. Skipping them meant every + // integer was decoded at one fixed signedness, so a signed TINYINT of -1 came + // back as 255 and a SMALLINT UNSIGNED of 65535 came back as -1. Nothing fails; + // the row is simply wrong, which is the worst way for this to be wrong. + column.unsigned = (reader.u16() & 0x0020) != 0; + return column; + } + + private static final class Column { + String name; + int type; + boolean binary; + boolean unsigned; + } + + // ---------------- packets ---------------- + + public void close() { + if(closed) { + return; + } + closed = true; + try { + sequence = 0; + sendPacket(new byte[]{0x01}); // COM_QUIT + } catch (IOException ignored) { + // the connection is going away regardless + } + wire.getConnection().close(); + } + + public boolean isClosed() { + return closed; + } + + private void checkOpen() throws IOException { + if(closed) { + throw new IOException("The MySQL connection is closed"); + } + } + + /** The most one MySQL packet can carry: the length field is 24 bits. */ + private static final int MAX_PACKET_BODY = 0xffffff; + + /** + * Sends a body, split across packets when it does not fit in one. + * + * A body of 16MB or more -- an ordinary large byte[] parameter -- has to go out + * as consecutive full-length packets with running sequence numbers. Writing the + * low 24 bits of the length and then the whole body left the server reading the + * remainder as the next packet's header, which does not fail: the connection is + * simply desynchronised from that point on, and every answer after it is + * nonsense. + * + * A body whose length is an exact multiple of the maximum ends with an empty + * packet, which is how the protocol says the sequence is over. + */ + private void sendPacket(byte[] body) throws IOException { + int offset = 0; + while(true) { + int chunk = body.length - offset; + if(chunk > MAX_PACKET_BODY) { + chunk = MAX_PACKET_BODY; + } + wire.writeByte(chunk & 0xff); + wire.writeByte((chunk >> 8) & 0xff); + wire.writeByte((chunk >> 16) & 0xff); + wire.writeByte(sequence++ & 0xff); + if(chunk > 0) { + wire.writeBytes(body, offset, chunk); + } + offset += chunk; + if(chunk < MAX_PACKET_BODY) { + break; + } + } + wire.flush(); + } + + private Packet readPacket() throws IOException { + int low = wire.read(); + if(low < 0) { + throw new IOException("The MySQL connection closed unexpectedly"); + } + int length = low | (wire.read() << 8) | (wire.read() << 16); + sequence = wire.read() + 1; + Packet packet = new Packet(); + packet.body = wire.readFully(length); + if(length == MAX_PACKET_BODY) { + // A full-length packet is continued by the next one, and the value only + // ends at a packet shorter than the maximum. Stopping at the first would + // hand the caller a truncated value and leave the following header to be + // read as data. + ByteArrayOutputStream all = new ByteArrayOutputStream(); + all.write(packet.body, 0, packet.body.length); + while(length == MAX_PACKET_BODY) { + int next = wire.read(); + if(next < 0) { + throw new IOException("The MySQL connection closed mid-packet"); + } + length = next | (wire.read() << 8) | (wire.read() << 16); + sequence = wire.read() + 1; + byte[] more = wire.readFully(length); + all.write(more, 0, more.length); + } + packet.body = all.toByteArray(); + } + if(packet.body.length == 0) { + throw new IOException("An empty MySQL packet"); + } + return packet; + } + + private static final class Packet { + byte[] body; + } + + private static IOException errorFrom(Packet packet, String sql) { + Reader reader = new Reader(packet.body); + reader.skip(1); + int code = reader.u16(); + String state = ""; + if(reader.remaining() > 0 && packet.body[3] == '#') { + reader.skip(1); + state = Wire.fromUtf8(reader.bytes(5)); + } + String message = Wire.fromUtf8(reader.rest()); + return new IOException("MySQL error " + code + + (state.length() == 0 ? "" : " " + state) + ": " + message + + (sql == null ? "" : " [" + sql + "]")); + } + + /** A cursor over one packet body. MySQL is little endian throughout. */ + private static final class Reader { + private final byte[] data; + private int at; + + Reader(byte[] data) { + this.data = data; + } + + int remaining() { + return data.length - at; + } + + void skip(int count) { + at += count; + } + + int u8() { + return data[at++] & 0xff; + } + + int u16() { + int value = (data[at] & 0xff) | ((data[at + 1] & 0xff) << 8); + at += 2; + return value; + } + + int i32() { + int value = (data[at] & 0xff) | ((data[at + 1] & 0xff) << 8) + | ((data[at + 2] & 0xff) << 16) | ((data[at + 3] & 0xff) << 24); + at += 4; + return value; + } + + long i64() { + long value = 0; + for(int iter = 0 ; iter < 8 ; iter++) { + value |= ((long)(data[at + iter] & 0xff)) << (iter * 8); + } + at += 8; + return value; + } + + byte[] bytes(int count) { + byte[] out = new byte[count]; + System.arraycopy(data, at, out, 0, count); + at += count; + return out; + } + + byte[] rest() { + return bytes(remaining()); + } + + String cString() { + int end = at; + while(end < data.length && data[end] != 0) { + end++; + } + String out = Wire.fromUtf8(data, at, end - at); + at = end + 1; + return out; + } + + /** A length-encoded integer; 0xfb is the NULL marker, returned as -1. */ + long lengthEncoded() { + int first = u8(); + if(first < 0xfb) { + return first; + } + if(first == 0xfb) { + return -1; + } + if(first == 0xfc) { + return u16(); + } + if(first == 0xfd) { + int value = (data[at] & 0xff) | ((data[at + 1] & 0xff) << 8) + | ((data[at + 2] & 0xff) << 16); + at += 3; + return value; + } + return i64(); + } + + byte[] lengthEncodedBytes() { + long length = lengthEncoded(); + return length < 0 ? null : bytes((int)length); + } + + /** + * DATE / DATETIME / TIMESTAMP, returned as an ISO string. A Java date type + * would have to be one the translated runtime also has, and every consumer + * of this data writes it into JSON anyway. + */ + String temporal() { + int length = u8(); + if(length == 0) { + return null; + } + int year = u16(); + int month = u8(); + int day = u8(); + StringBuilder out = new StringBuilder(); + pad(out, year, 4).append('-'); + pad(out, month, 2).append('-'); + pad(out, day, 2); + if(length > 4) { + int hour = u8(); + int minute = u8(); + int second = u8(); + out.append(' '); + pad(out, hour, 2).append(':'); + pad(out, minute, 2).append(':'); + pad(out, second, 2); + if(length > 7) { + int micros = i32(); + out.append('.'); + pad(out, micros, 6); + } + } + return out.toString(); + } + + String time() { + int length = u8(); + if(length == 0) { + return "00:00:00"; + } + boolean negative = u8() != 0; + int days = i32(); + int hour = u8(); + int minute = u8(); + int second = u8(); + StringBuilder out = new StringBuilder(); + if(negative) { + out.append('-'); + } + pad(out, days * 24 + hour, 2).append(':'); + pad(out, minute, 2).append(':'); + pad(out, second, 2); + if(length > 8) { + int micros = i32(); + out.append('.'); + pad(out, micros, 6); + } + return out.toString(); + } + + private static StringBuilder pad(StringBuilder out, int value, int width) { + String text = String.valueOf(value); + for(int iter = text.length() ; iter < width ; iter++) { + out.append('0'); + } + return out.append(text); + } + } + + private static void writeIntLE(ByteArrayOutputStream out, int value) { + out.write(value & 0xff); + out.write((value >> 8) & 0xff); + out.write((value >> 16) & 0xff); + out.write((value >> 24) & 0xff); + } + + private static void writeLongLE(ByteArrayOutputStream out, long value) { + for(int iter = 0 ; iter < 8 ; iter++) { + out.write((int)((value >> (iter * 8)) & 0xff)); + } + } + + private static void writeCString(ByteArrayOutputStream out, String value) { + byte[] data = Wire.utf8(value); + out.write(data, 0, data.length); + out.write(0); + } + + private static void writeLengthEncoded(ByteArrayOutputStream out, byte[] data) { + int length = data == null ? 0 : data.length; + if(length < 251) { + out.write(length); + } else if(length < 65536) { + out.write(0xfc); + out.write(length & 0xff); + out.write((length >> 8) & 0xff); + } else if(length <= MAX_PACKET_BODY) { + out.write(0xfd); + out.write(length & 0xff); + out.write((length >> 8) & 0xff); + out.write((length >> 16) & 0xff); + } else { + // 0xfd carries three bytes of length and stops at 0xffffff. A larger + // value needs 0xfe and eight, and writing the small form for it sends a + // truncated length: the server then reads the rest of the value as the + // next thing in the packet. Fragmenting the packet does not help, because + // this prefix is inside it. + out.write(0xfe); + for(int iter = 0 ; iter < 8 ; iter++) { + out.write((int)((long)length >> (8 * iter)) & 0xff); + } + } + if(length > 0) { + out.write(data, 0, length); + } + } + + private static byte[] concat(byte[] a, byte[] b) { + byte[] out = new byte[a.length + b.length]; + System.arraycopy(a, 0, out, 0, a.length); + System.arraycopy(b, 0, out, a.length, b.length); + return out; + } + + private static byte[] xor(byte[] a, byte[] b) { + byte[] out = new byte[a.length]; + for(int iter = 0 ; iter < a.length ; iter++) { + out[iter] = (byte)(a[iter] ^ b[iter % b.length]); + } + return out; + } + + private static byte[] trimTrailingNul(byte[] data) { + int length = data.length; + while(length > 0 && data[length - 1] == 0) { + length--; + } + byte[] out = new byte[length]; + System.arraycopy(data, 0, out, 0, length); + return out; + } +} diff --git a/vm/backend/src/com/codename1/backend/sql/Postgres.java b/vm/backend/src/com/codename1/backend/sql/Postgres.java new file mode 100644 index 00000000000..ba96a87ebbb --- /dev/null +++ b/vm/backend/src/com/codename1/backend/sql/Postgres.java @@ -0,0 +1,756 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.sql; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import com.codename1.backend.Base64; +import com.codename1.backend.Crypto; +import com.codename1.backend.Tcp; + +/** + * A PostgreSQL client speaking the v3 frontend/backend protocol directly. + * + * Written rather than wrapped because there is nothing to wrap: JDBC needs a + * driver manager, a class loader and reflection, none of which a translated + * server binary has. The protocol is small, stable (v3 has been the wire format + * since 7.4) and documented, so the honest option is to speak it. The same source + * runs on both targets because it is built on {@link Tcp}, which each target + * implements. + * + * Three decisions worth stating: + * + * - **The extended query protocol, always.** Simple Query would be fewer round + * trips, but it has no parameters, and a client with no way to bind a value is + * a client whose users concatenate SQL. Parse/Bind/Execute is what makes + * `query(sql, params)` safe by construction. + * - **Text format for parameters and results.** The binary format saves parsing + * at the cost of a per-type encoder on both sides, and gets subtly wrong for + * the types nobody tested. Text is what psql sends. + * - **SCRAM-SHA-256 is verified in both directions.** The server's final message + * proves it knew the stored key; skipping that check (which a client can do and + * still connect successfully) leaves the handshake open to a server that only + * pretends to be PostgreSQL. + */ +public final class Postgres { + /** Message types the backend sends that this client acts on. */ + private static final int AUTHENTICATION = 'R'; + private static final int ERROR_RESPONSE = 'E'; + private static final int ROW_DESCRIPTION = 'T'; + private static final int DATA_ROW = 'D'; + private static final int COMMAND_COMPLETE = 'C'; + private static final int READY_FOR_QUERY = 'Z'; + + private final Wire wire; + private final String user; + private final String password; + private boolean closed; + + private Postgres(Wire wire, String user, String password) { + this.wire = wire; + this.user = user; + this.password = password; + } + + /** + * Connects, negotiates TLS when asked, authenticates, and returns a session + * ready for queries. + * + * `sslMode` is "require", "prefer" or "disable". "prefer" exists because it is + * what a local development database usually needs and a managed one usually + * forbids; "require" fails rather than falling back, which is the only setting + * that means anything against an attacker. + */ + public static Postgres connect(String host, int port, String database, String user, + String password, String sslMode, String caFile, int timeoutMillis) throws IOException { + Tcp connection = Tcp.connect(host, port <= 0 ? 5432 : port, timeoutMillis); + try { + Wire wire = new Wire(connection); + if(!"disable".equals(sslMode)) { + boolean offered = requestTls(wire); + boolean required = "require".equals(sslMode); + if(!offered && required) { + throw new IOException("The server at " + host + + " refused TLS and sslmode=require"); + } + if(offered) { + try { + connection.startTls(host, caFile); + } catch (IOException err) { + if(required) { + throw err; + } + // sslmode=prefer, so this falls back -- but it says so. + // A silent downgrade, or an unverified session presented as + // a verified one, is how a connection ends up looking + // encrypted and being neither authenticated nor private. + throw new IOException("TLS to " + host + " could not be " + + "verified (" + err.getMessage() + "). Point " + + "sslrootcert at the server's CA, or set " + + "sslmode=disable to connect in the clear " + + "deliberately."); + } + } + } + Postgres session = new Postgres(wire, user, password); + session.startup(database, user); + return session; + } catch (IOException err) { + connection.close(); + throw err; + } + } + + /** + * The SSLRequest packet. It is not a normal message -- no type byte, and the + * reply is a single character rather than a framed message -- because it is + * sent before the protocol proper begins. + */ + private static boolean requestTls(Wire wire) throws IOException { + wire.writeIntBE(8); + wire.writeIntBE(80877103); // 1234 << 16 | 5679 + wire.flush(); + int answer = wire.read(); + if(answer == 'S') { + return true; + } + if(answer == 'N') { + return false; + } + throw new IOException("The server did not answer the TLS request (got " + answer + ")"); + } + + private void startup(String database, String user) throws IOException { + ByteArrayOutputStream body = new ByteArrayOutputStream(); + writeInt(body, 196608); // protocol 3.0 + writeCString(body, "user"); + writeCString(body, user); + if(database != null && database.length() > 0) { + writeCString(body, "database"); + writeCString(body, database); + } + // Errors come back in whatever the server's locale says otherwise, which + // makes a failure unreadable in a log written by someone else. + writeCString(body, "client_encoding"); + writeCString(body, "UTF8"); + body.write(0); + byte[] payload = body.toByteArray(); + wire.writeIntBE(payload.length + 4); + wire.writeBytes(payload); + wire.flush(); + authenticate(); + // Everything from here to ReadyForQuery is parameter status, the backend + // key and notices: none of it changes what this client does. + readUntilReady(); + } + + private void authenticate() throws IOException { + while(true) { + Message message = readMessage(); + if(message.type == ERROR_RESPONSE) { + throw errorFrom(message); + } + if(message.type != AUTHENTICATION) { + throw new IOException("Expected an authentication message, got '" + + (char)message.type + "'"); + } + int method = intAt(message.body, 0); + if(method == 0) { + return; // authentication complete + } + if(method == 3) { + sendPasswordMessage(Wire.utf8(password == null ? "" : password)); + continue; + } + if(method == 5) { + byte[] salt = new byte[4]; + System.arraycopy(message.body, 4, salt, 0, 4); + sendPasswordMessage(Wire.utf8(md5Password(user, password, salt))); + continue; + } + if(method == 10) { + scram(message); + continue; + } + throw new IOException("Unsupported authentication method " + method + + "; this client speaks SCRAM-SHA-256, md5 and cleartext"); + } + } + + /** + * PostgreSQL's md5 method: md5(md5(password + user) as hex, then salted). + * Deprecated by PostgreSQL itself, and supported here only because servers + * configured for it are still deployed. + */ + private static String md5Password(String user, String password, byte[] salt) { + String inner = hex(Crypto.md5(Wire.utf8((password == null ? "" : password) + user))); + byte[] withSalt = concat(Wire.utf8(inner), salt); + return "md5" + hex(Crypto.md5(withSalt)); + } + + /** + * SCRAM-SHA-256 (RFC 7677), the default since PostgreSQL 10 and the only + * method a managed instance normally offers. + * + * The server's final message is checked. A client that skips it authenticates + * itself TO the server and learns nothing about who it is talking to, which + * defeats the mutual half of the mechanism. + */ + private void scram(Message advertised) throws IOException { + if(!mechanismsInclude(advertised.body, "SCRAM-SHA-256")) { + throw new IOException("The server offers no SCRAM-SHA-256; this client " + + "does not implement the channel-binding variants"); + } + String clientNonce = Base64.encode(Crypto.randomBytes(18)); + String clientFirstBare = "n=,r=" + clientNonce; + byte[] initial = Wire.utf8("n,," + clientFirstBare); + + ByteArrayOutputStream body = new ByteArrayOutputStream(); + writeCString(body, "SCRAM-SHA-256"); + writeInt(body, initial.length); + body.write(initial, 0, initial.length); + sendMessage('p', body.toByteArray()); + + Message serverFirstMessage = readMessage(); + if(serverFirstMessage.type == ERROR_RESPONSE) { + throw errorFrom(serverFirstMessage); + } + if(serverFirstMessage.type != AUTHENTICATION || intAt(serverFirstMessage.body, 0) != 11) { + throw new IOException("Expected a SASL continue message"); + } + String serverFirst = Wire.fromUtf8(serverFirstMessage.body, 4, + serverFirstMessage.body.length - 4); + String nonce = field(serverFirst, 'r'); + String saltText = field(serverFirst, 's'); + String iterationText = field(serverFirst, 'i'); + if(nonce == null || saltText == null || iterationText == null + || !nonce.startsWith(clientNonce)) { + // A nonce that does not extend ours means the exchange was replayed or + // rewritten; there is nothing to continue. + throw new IOException("The server's SCRAM message is malformed or replayed"); + } + byte[] salt = Base64.decode(saltText); + if(salt == null) { + throw new IOException("The server's SCRAM salt is not base64"); + } + int iterations = parseInt(iterationText, -1); + if(iterations < 1) { + throw new IOException("The server's SCRAM iteration count is not a number"); + } + + // The password goes into PBKDF2 as its raw UTF-8, WITHOUT SASLprep, and + // that is a deliberate limitation rather than an oversight. SASLprep + // (RFC 4013) is a stringprep profile whose mapping step is NFKC, and + // there is no Normalizer on this platform -- this class is translated + // for the packaged server, so it may only use what vm/JavaAPI and + // CLDC11 define, and neither has java.text. Implementing the part that + // needs no Unicode tables would make things WORSE, not better: + // PostgreSQL falls back to the raw password whenever its own saslprep + // rejects the input, so a half-prepared password would stop matching + // verifiers that work today. Printable ASCII -- which SASLprep leaves + // untouched -- is therefore correct here; a password that SASLprep + // would normalise is rejected, and has to be set in ASCII or + // authenticated by another method. + byte[] saltedPassword = Crypto.pbkdf2Sha256( + Wire.utf8(password == null ? "" : password), salt, iterations, 32); + byte[] clientKey = Crypto.hmacSha256(saltedPassword, Wire.utf8("Client Key")); + byte[] storedKey = Crypto.sha256(clientKey); + String clientFinalWithoutProof = "c=biws,r=" + nonce; // biws is base64("n,,") + byte[] authMessage = Wire.utf8(clientFirstBare + "," + serverFirst + "," + + clientFinalWithoutProof); + byte[] clientSignature = Crypto.hmacSha256(storedKey, authMessage); + byte[] proof = new byte[clientKey.length]; + for(int iter = 0 ; iter < proof.length ; iter++) { + proof[iter] = (byte)(clientKey[iter] ^ clientSignature[iter]); + } + sendMessage('p', Wire.utf8(clientFinalWithoutProof + ",p=" + Base64.encode(proof))); + + Message finalMessage = readMessage(); + if(finalMessage.type == ERROR_RESPONSE) { + throw errorFrom(finalMessage); + } + if(finalMessage.type != AUTHENTICATION || intAt(finalMessage.body, 0) != 12) { + throw new IOException("Expected the SASL final message"); + } + String serverFinal = Wire.fromUtf8(finalMessage.body, 4, finalMessage.body.length - 4); + String signatureText = field(serverFinal, 'v'); + byte[] serverKey = Crypto.hmacSha256(saltedPassword, Wire.utf8("Server Key")); + byte[] expected = Crypto.hmacSha256(serverKey, authMessage); + byte[] actual = signatureText == null ? null : Base64.decode(signatureText); + if(actual == null || !Crypto.equalsConstantTime(expected, actual)) { + throw new IOException("The server failed the SCRAM signature check; it does " + + "not hold the credentials it claims to"); + } + } + + private static boolean mechanismsInclude(byte[] body, String wanted) { + int at = 4; + while(at < body.length) { + int end = at; + while(end < body.length && body[end] != 0) { + end++; + } + if(end == at) { + return false; // the empty string terminates the list + } + if(wanted.equals(Wire.fromUtf8(body, at, end - at))) { + return true; + } + at = end + 1; + } + return false; + } + + /** One `k=value` field out of a SCRAM message. */ + private static String field(String message, char key) { + int at = 0; + while(at < message.length()) { + int end = message.indexOf(',', at); + if(end < 0) { + end = message.length(); + } + if(end - at > 2 && message.charAt(at) == key && message.charAt(at + 1) == '=') { + return message.substring(at + 2, end); + } + at = end + 1; + } + return null; + } + + private void sendPasswordMessage(byte[] password) throws IOException { + byte[] body = new byte[password.length + 1]; + System.arraycopy(password, 0, body, 0, password.length); + sendMessage('p', body); + } + + // ---------------- queries ---------------- + + /** Runs a statement that returns no rows, and returns the number affected. */ + public int execute(String sql, Object[] params) throws IOException { + Result result = run(sql, params); + return result.affected; + } + + /** + * Runs a query and returns each row as a column-name to value map, with the + * SAME value types the SQLite path produces: Long, Double, String, byte[] or + * null. A handler must not be able to tell which engine answered it. + */ + public List query(String sql, Object[] params) throws IOException { + return run(sql, params).rows; + } + + private Result run(String sql, Object[] params) throws IOException { + checkOpen(); + // Parse into the unnamed statement, bind the unnamed portal, describe, + // execute, sync. One round trip for the lot. + ByteArrayOutputStream parse = new ByteArrayOutputStream(); + writeCString(parse, ""); + writeCString(parse, sql); + writeShort(parse, 0); // let the server infer every parameter type + stageMessage('P', parse.toByteArray()); + + ByteArrayOutputStream bind = new ByteArrayOutputStream(); + writeCString(bind, ""); // portal + writeCString(bind, ""); // statement + writeShort(bind, 0); // parameter formats: none given, so all text + int count = params == null ? 0 : params.length; + writeShort(bind, count); + for(int iter = 0 ; iter < count ; iter++) { + byte[] encoded = encodeParameter(params[iter]); + if(encoded == null) { + writeInt(bind, -1); // SQL NULL, which is not the empty string + } else { + writeInt(bind, encoded.length); + bind.write(encoded, 0, encoded.length); + } + } + writeShort(bind, 0); // result formats: none given, so all text + stageMessage('B', bind.toByteArray()); + + ByteArrayOutputStream describe = new ByteArrayOutputStream(); + describe.write('P'); + writeCString(describe, ""); + stageMessage('D', describe.toByteArray()); + + ByteArrayOutputStream execute = new ByteArrayOutputStream(); + writeCString(execute, ""); // portal + writeInt(execute, 0); // no row limit + stageMessage('E', execute.toByteArray()); + + stageMessage('S', new byte[0]); + wire.flush(); + + return collect(sql); + } + + private Result collect(String sql) throws IOException { + Result result = new Result(); + String[] names = null; + int[] types = null; + IOException failure = null; + while(true) { + Message message = readMessage(); + switch(message.type) { + case ROW_DESCRIPTION: { + int columns = shortAt(message.body, 0); + names = new String[columns]; + types = new int[columns]; + int at = 2; + for(int iter = 0 ; iter < columns ; iter++) { + int end = at; + while(end < message.body.length && message.body[end] != 0) { + end++; + } + names[iter] = Wire.fromUtf8(message.body, at, end - at); + at = end + 1; + // table oid (4), column number (2), then the type oid (4) + types[iter] = intAt(message.body, at + 6); + at += 18; // + type size (2), modifier (4), format (2) + } + break; + } + case DATA_ROW: { + int columns = shortAt(message.body, 0); + Map row = new LinkedHashMap(); + int at = 2; + for(int iter = 0 ; iter < columns ; iter++) { + int length = intAt(message.body, at); + at += 4; + Object value; + if(length < 0) { + value = null; + } else { + value = decode(Wire.fromUtf8(message.body, at, length), + types == null ? 0 : types[iter]); + at += length; + } + row.put(names == null ? String.valueOf(iter) : names[iter], value); + } + result.rows.add(row); + break; + } + case COMMAND_COMPLETE: { + String tag = Wire.fromUtf8(message.body, 0, message.body.length - 1); + // A COMMIT on a transaction the server has already marked + // aborted completes with the ROLLBACK tag rather than an + // error -- which happens whenever a statement failed and the + // transaction body caught it and carried on. Every change in + // the transaction is discarded, and reading only the row + // count out of this reports that as a successful commit and + // hands the caller the body's result. + if("ROLLBACK".equals(tag) && "COMMIT".equalsIgnoreCase(sql.trim())) { + failure = new IOException("COMMIT rolled the transaction back: the " + + "server had already marked it aborted, so nothing in it " + + "was applied"); + } + result.affected = affectedFrom(tag); + break; + } + case ERROR_RESPONSE: + // Not thrown here: the server still owes us a ReadyForQuery, and + // leaving it unread desynchronises every later statement. + failure = errorFrom(message, sql); + break; + case READY_FOR_QUERY: + if(failure != null) { + throw failure; + } + return result; + default: + break; // ParseComplete, BindComplete, NoData, notices, parameter status + } + } + } + + /** + * "INSERT 0 3", "UPDATE 2", "DELETE 1", "SELECT 7": the count is the last + * word, and INSERT is the one with an oid before it. + */ + private static int affectedFrom(String tag) { + int space = tag.lastIndexOf(' '); + return space < 0 ? 0 : parseInt(tag.substring(space + 1), 0); + } + + /** + * Parameters go out as text, so this is a rendering rather than an encoding. + * A byte[] becomes a bytea hex literal, which is what the server expects in + * text format; everything else is its ordinary string form. + */ + private static byte[] encodeParameter(Object value) { + if(value == null) { + return null; + } + if(value instanceof byte[]) { + return Wire.utf8("\\x" + hex((byte[])value)); + } + if(value instanceof Boolean) { + return Wire.utf8(((Boolean)value).booleanValue() ? "t" : "f"); + } + return Wire.utf8(String.valueOf(value)); + } + + /** + * Maps a text-format value to the same Java types the SQLite path returns. + * The OIDs are the stable built-in ones from pg_type; a type this does not + * know stays a String, which is what the server sent. + */ + private static Object decode(String text, int typeOid) { + switch(typeOid) { + case 16: // bool + return Long.valueOf("t".equals(text) ? 1 : 0); + case 20: // int8 + case 21: // int2 + case 23: // int4 + case 26: // oid + try { + return Long.valueOf(Long.parseLong(text.trim())); + } catch (NumberFormatException err) { + return text; + } + case 700: // float4 + case 701: // float8 + try { + return Double.valueOf(Double.parseDouble(text.trim())); + } catch (NumberFormatException err) { + return text; + } + case 1700: // numeric + // NOT a double. numeric is arbitrary precision, and + // Double.parseDouble does not fail on the values it cannot + // hold: 1e999 comes back as infinity, which Json then writes + // as null, and a merely large numeric comes back quietly + // rounded. Both report a successful query with a value the + // database does not hold. The exact text is what the server + // sent, so that is what the caller gets -- matching DECIMAL + // on the MySQL path, which is the same kind of column. + return text; + case 17: { // bytea, sent as \x48656c6c6f + if(text.length() >= 2 && text.charAt(0) == '\\' && text.charAt(1) == 'x') { + byte[] out = unhex(text.substring(2)); + if(out != null) { + return out; + } + } + return text; + } + default: + return text; + } + } + + // ---------------- plumbing ---------------- + + public void close() { + if(closed) { + return; + } + closed = true; + try { + // Terminate, so the server logs a clean disconnect rather than a + // broken connection for every pooled session that ends. + stageMessage('X', new byte[0]); + wire.flush(); + } catch (IOException ignored) { + // the connection is going away regardless + } + wire.getConnection().close(); + } + + public boolean isClosed() { + return closed; + } + + private void checkOpen() throws IOException { + if(closed) { + throw new IOException("The PostgreSQL connection is closed"); + } + } + + private void readUntilReady() throws IOException { + while(true) { + Message message = readMessage(); + if(message.type == ERROR_RESPONSE) { + throw errorFrom(message); + } + if(message.type == READY_FOR_QUERY) { + return; + } + } + } + + private void stageMessage(int type, byte[] body) { + wire.writeByte(type); + wire.writeIntBE(body.length + 4); + wire.writeBytes(body); + } + + private void sendMessage(int type, byte[] body) throws IOException { + stageMessage(type, body); + wire.flush(); + } + + private Message readMessage() throws IOException { + int type = wire.read(); + if(type < 0) { + throw new IOException("The PostgreSQL connection closed unexpectedly"); + } + int length = wire.readIntBE(); + if(length < 4) { + throw new IOException("A PostgreSQL message claims length " + length); + } + Message message = new Message(); + message.type = type; + message.body = wire.readFully(length - 4); + return message; + } + + private static IOException errorFrom(Message message) { + return errorFrom(message, null); + } + + /** + * An ErrorResponse is a set of typed fields; 'M' is the human message, 'C' the + * SQLSTATE. Both go into the exception, because the SQLSTATE is what tells a + * caller apart a unique-violation from a syntax error. + */ + private static IOException errorFrom(Message message, String sql) { + String detail = null; + String state = null; + int at = 0; + while(at < message.body.length && message.body[at] != 0) { + int field = message.body[at]; + int end = at + 1; + while(end < message.body.length && message.body[end] != 0) { + end++; + } + String value = Wire.fromUtf8(message.body, at + 1, end - at - 1); + if(field == 'M') { + detail = value; + } else if(field == 'C') { + state = value; + } + at = end + 1; + } + return new IOException("PostgreSQL error" + + (state == null ? "" : " " + state) + ": " + + (detail == null ? "unknown" : detail) + + (sql == null ? "" : " [" + sql + "]")); + } + + private static final class Message { + int type; + byte[] body; + } + + private static final class Result { + final List rows = new ArrayList(); + int affected; + } + + private static int intAt(byte[] data, int offset) { + return ((data[offset] & 0xff) << 24) | ((data[offset + 1] & 0xff) << 16) + | ((data[offset + 2] & 0xff) << 8) | (data[offset + 3] & 0xff); + } + + private static int shortAt(byte[] data, int offset) { + return ((data[offset] & 0xff) << 8) | (data[offset + 1] & 0xff); + } + + private static void writeInt(ByteArrayOutputStream out, int value) { + out.write((value >> 24) & 0xff); + out.write((value >> 16) & 0xff); + out.write((value >> 8) & 0xff); + out.write(value & 0xff); + } + + private static void writeShort(ByteArrayOutputStream out, int value) { + out.write((value >> 8) & 0xff); + out.write(value & 0xff); + } + + private static void writeCString(ByteArrayOutputStream out, String value) { + byte[] data = Wire.utf8(value); + out.write(data, 0, data.length); + out.write(0); + } + + private static byte[] concat(byte[] a, byte[] b) { + byte[] out = new byte[a.length + b.length]; + System.arraycopy(a, 0, out, 0, a.length); + System.arraycopy(b, 0, out, a.length, b.length); + return out; + } + + private static final char[] HEX = "0123456789abcdef".toCharArray(); + + static String hex(byte[] data) { + StringBuilder out = new StringBuilder(data.length * 2); + for(int iter = 0 ; iter < data.length ; iter++) { + out.append(HEX[(data[iter] >> 4) & 0xf]).append(HEX[data[iter] & 0xf]); + } + return out.toString(); + } + + private static byte[] unhex(String text) { + if((text.length() % 2) != 0) { + return null; + } + byte[] out = new byte[text.length() / 2]; + for(int iter = 0 ; iter < out.length ; iter++) { + int high = digit(text.charAt(iter * 2)); + int low = digit(text.charAt(iter * 2 + 1)); + if(high < 0 || low < 0) { + return null; + } + out[iter] = (byte)((high << 4) | low); + } + return out; + } + + private static int digit(char c) { + if(c >= '0' && c <= '9') { + return c - '0'; + } + if(c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if(c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + + private static int parseInt(String value, int fallback) { + try { + return Integer.parseInt(value.trim()); + } catch (NumberFormatException err) { + return fallback; + } + } +} diff --git a/vm/backend/src/com/codename1/backend/sql/Wire.java b/vm/backend/src/com/codename1/backend/sql/Wire.java new file mode 100644 index 00000000000..1a77f0bc3e9 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/sql/Wire.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.sql; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; + +import com.codename1.backend.Tcp; + +/** + * Buffered framing over a {@link Tcp} connection, shared by the PostgreSQL and + * MySQL clients. + * + * Both protocols are length-prefixed binary, and both are read a packet at a + * time, so an unbuffered read per field would be one system call per integer. The + * read buffer here is what makes a row decode a memory operation. + * + * The two protocols disagree about byte order -- PostgreSQL is big endian and + * MySQL little endian -- so both are provided rather than picking one and having + * a client remember to swap. + */ +final class Wire { + private final Tcp connection; + private final byte[] buffer = new byte[16384]; + private int position; + private int limit; + private final ByteArrayOutputStream out = new ByteArrayOutputStream(1024); + + Wire(Tcp connection) { + this.connection = connection; + } + + Tcp getConnection() { + return connection; + } + + // ---------------- reading ---------------- + + /** One byte, or -1 at end of stream. */ + int read() throws IOException { + if(position >= limit && !fill()) { + return -1; + } + return buffer[position++] & 0xff; + } + + /** Exactly `length` bytes, or an IOException: a short packet is a protocol error. */ + byte[] readFully(int length) throws IOException { + byte[] target = new byte[length]; + readFully(target, 0, length); + return target; + } + + void readFully(byte[] target, int offset, int length) throws IOException { + int at = 0; + while(at < length) { + if(position >= limit && !fill()) { + throw new IOException("The connection closed after " + at + " of " + + length + " bytes"); + } + int available = limit - position; + int take = available < length - at ? available : length - at; + System.arraycopy(buffer, position, target, offset + at, take); + position += take; + at += take; + } + } + + /** Discards `length` bytes without allocating for them. */ + void skip(int length) throws IOException { + int remaining = length; + while(remaining > 0) { + if(position >= limit && !fill()) { + throw new IOException("The connection closed while skipping"); + } + int available = limit - position; + int take = available < remaining ? available : remaining; + position += take; + remaining -= take; + } + } + + int readIntBE() throws IOException { + byte[] b = readFully(4); + return ((b[0] & 0xff) << 24) | ((b[1] & 0xff) << 16) | ((b[2] & 0xff) << 8) | (b[3] & 0xff); + } + + int readShortBE() throws IOException { + byte[] b = readFully(2); + return ((b[0] & 0xff) << 8) | (b[1] & 0xff); + } + + private boolean fill() throws IOException { + position = 0; + limit = 0; + int n = connection.read(buffer, 0, buffer.length); + if(n <= 0) { + return false; + } + limit = n; + return true; + } + + // ---------------- writing ---------------- + + void writeByte(int value) { + out.write(value & 0xff); + } + + void writeBytes(byte[] data) { + if(data != null) { + out.write(data, 0, data.length); + } + } + + void writeBytes(byte[] data, int offset, int length) { + out.write(data, offset, length); + } + + void writeShortBE(int value) { + out.write((value >> 8) & 0xff); + out.write(value & 0xff); + } + + void writeIntBE(int value) { + out.write((value >> 24) & 0xff); + out.write((value >> 16) & 0xff); + out.write((value >> 8) & 0xff); + out.write(value & 0xff); + } + + void writeShortLE(int value) { + out.write(value & 0xff); + out.write((value >> 8) & 0xff); + } + + void writeIntLE(int value) { + out.write(value & 0xff); + out.write((value >> 8) & 0xff); + out.write((value >> 16) & 0xff); + out.write((value >> 24) & 0xff); + } + + void writeLongLE(long value) { + for(int iter = 0 ; iter < 8 ; iter++) { + out.write((int)((value >> (iter * 8)) & 0xff)); + } + } + + /** A NUL-terminated string, which is how both protocols carry names. */ + void writeCString(String value) { + writeBytes(utf8(value)); + out.write(0); + } + + /** How many bytes are staged but not yet sent. */ + int pending() { + return out.size(); + } + + byte[] take() { + byte[] data = out.toByteArray(); + out.reset(); + return data; + } + + /** Sends everything staged and clears the buffer. */ + void flush() throws IOException { + byte[] data = take(); + if(data.length > 0) { + connection.write(data, 0, data.length); + } + } + + static byte[] utf8(String value) { + if(value == null) { + return new byte[0]; + } + try { + return value.getBytes("UTF-8"); + } catch (UnsupportedEncodingException err) { + throw new IllegalStateException("UTF-8 is missing"); + } + } + + static String fromUtf8(byte[] data, int offset, int length) { + try { + return new String(data, offset, length, "UTF-8"); + } catch (UnsupportedEncodingException err) { + throw new IllegalStateException("UTF-8 is missing"); + } + } + + static String fromUtf8(byte[] data) { + return data == null ? null : fromUtf8(data, 0, data.length); + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendDatabaseTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendDatabaseTest.java new file mode 100644 index 00000000000..01d4bc13926 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BackendDatabaseTest.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * The database layer, against real engines, on both runtimes. + * + * vm/backend/demo/dbcheck runs one body of assertions -- types, binding, + * transactions, rollback, error recovery -- and the Database facade claims a + * handler cannot tell which engine answered it. The only way to hold that claim + * is to run the same body against every engine and require the same result, so + * that is what this does, on the translated binary AND on the local Java SE arm. + * + * SQLite always runs; it needs nothing installed. PostgreSQL and MySQL run when + * CN1_DBCHECK_POSTGRES / CN1_DBCHECK_MYSQL name a server (CI supplies both as + * service containers). Their absence is a skip on a developer machine, and a + * FAILURE where the backend is required, so a CI runner that quietly lost its + * database cannot report green. + */ +class BackendDatabaseTest { + + @Test + @DisplayName("every configured database engine answers the same on both runtimes") + void everyEngineAgrees() throws Exception { + if (CompilerHelper.isWindows()) { + Assumptions.abort("the server-side backend is POSIX-only for now"); + } + BackendTestSupport.require(Files.isDirectory(BackendTestSupport.backendDir()), + "vm/backend is not present"); + Path jdk8 = BackendTestSupport.findJdk8(); + BackendTestSupport.require(jdk8 != null, "no JDK 8 available to build the backend"); + + List urls = new ArrayList(); + urls.add(":memory:"); + addIfSet(urls, "CN1_DBCHECK_POSTGRES"); + addIfSet(urls, "CN1_DBCHECK_MYSQL"); + if (urls.size() == 1 && BackendTestSupport.isRequired()) { + fail("CN1_DBCHECK_POSTGRES and CN1_DBCHECK_MYSQL are unset, so only SQLite " + + "was exercised; the server engines are the ones with a wire " + + "protocol to get wrong"); + } + + Path work = Files.createTempDirectory("backend-dbcheck"); + Path binary = work.resolve("dbcheck"); + String failure = BackendTestSupport.build("DbCheck", "demo/dbcheck", binary, jdk8); + if (failure != null) { + BackendTestSupport.skipOrFail(failure); + } + + for (String url : urls) { + String translated = runTranslated(binary, url); + assertOk("the translated runtime", url, translated); + String local = runLocal(url, jdk8); + assertOk("the local Java SE runtime", url, local); + // Not just "both said OK": the same number of checks has to have run, + // or one runtime skipping half of them would still pass. + assertEquals(passedCount(translated), passedCount(local), + "the two runtimes ran a different number of checks against " + + redact(url) + "\n--- translated ---\n" + translated + + "\n--- local ---\n" + local); + } + } + + private static void addIfSet(List urls, String name) { + String value = System.getenv(name); + if (value != null && value.length() > 0) { + urls.add(value); + } + } + + private static String runTranslated(Path binary, String url) throws Exception { + Map env = new HashMap(); + env.put("CN1_DBCHECK_URL", url); + ProcessBuilder run = new ProcessBuilder(binary.toString()); + run.environment().putAll(env); + run.redirectErrorStream(true); + Process p = run.start(); + String output = BackendTestSupport.readFully(p.getInputStream()); + if (!p.waitFor(5, TimeUnit.MINUTES)) { + p.destroyForcibly(); + fail("the translated dbcheck did not finish:\n" + output); + } + return output; + } + + private static String runLocal(String url, Path jdk8) throws Exception { + Map env = new HashMap(); + env.put("CN1_BACKEND_JAVA", System.getProperty("java.home")); + env.put("CN1_BACKEND_DEMO", "demo/dbcheck"); + env.put("CN1_BACKEND_JDBC_JARS", BackendTestSupport.jdbcJars()); + env.put("CN1_DBCHECK_URL", url); + env.put("JDK_8_HOME", jdk8.toString()); + int[] status = new int[1]; + return BackendTestSupport.runBackendScript( + new ArrayList(Arrays.asList("./run-javase.sh", "com.demo.DbCheck")), + env, 600, status); + } + + private static void assertOk(String which, String url, String output) { + assertTrue(output.indexOf("DBCHECK OK") >= 0, + which + " failed against " + redact(url) + ":\n" + output); + assertTrue(passedCount(output) >= 20, + which + " ran only " + passedCount(output) + " checks against " + + redact(url) + ":\n" + output); + } + + /** A URL carries a password, and this output ends up in a CI log. */ + private static String redact(String url) { + int at = url.indexOf('@'); + int scheme = url.indexOf("://"); + if (at < 0 || scheme < 0) { + return url; + } + return url.substring(0, scheme + 3) + "***" + url.substring(at); + } + + private static int passedCount(String output) { + int at = output.indexOf("passed="); + if (at < 0) { + return -1; + } + int end = output.indexOf(' ', at); + try { + return Integer.parseInt(output.substring(at + "passed=".length(), + end < 0 ? output.length() : end).trim()); + } catch (NumberFormatException err) { + return -1; + } + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendHttpIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendHttpIntegrationTest.java new file mode 100644 index 00000000000..d5e2d9677d7 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BackendHttpIntegrationTest.java @@ -0,0 +1,2456 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.Socket; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.security.cert.X509Certificate; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Drives the translated server-side binary over a real socket. + * + * The functional half could be written against curl; the protocol half cannot. + * Pipelining, a request that carries both Content-Length and Transfer-Encoding, + * an obsolete folded header -- no ordinary client will send any of those, and they + * are exactly the inputs a server has to get right. So every assertion here goes + * through a raw socket and reads the bytes back. + * + * The suite builds vm/backend once and runs one server for the class. It SKIPS + * rather than fails when the toolchain is missing (no JDK 8, no clang, Windows), + * because a machine that cannot build the binary has nothing to say about whether + * the binary is correct. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class BackendHttpIntegrationTest { + + private static Process server; + private static int port; + + private static Process tlsServer; + private static int tlsPort; + + /** + * A third server, pinned to ONE virtual-thread host by CN1_WORKERS=1, so a + * test can keep that host continuously busy. With several hosts the traffic + * and the silent connection may land on different ones and the test would + * prove nothing some of the time, which is worse than not having it. + */ + private static Process busyServer; + private static int busyPort; + private static Process smallUploadServer; + private static int smallUploadPort; + private static Path smallUploadLog; + + /** Larger than any plausible socket send buffer, so a slow reader stalls the write. */ + private static final int HUGE_BYTES = 8 * 1024 * 1024; + private static Path work; + private static String skipReason; + + @BeforeAll + void startServer() throws Exception { + if (CompilerHelper.isWindows()) { + skipReason = "the server-side backend is POSIX-only for now"; + BackendTestSupport.skipOrFail(skipReason); + } + Path backend = Paths.get("..", "backend").normalize().toAbsolutePath(); + BackendTestSupport.require(Files.isDirectory(backend), "vm/backend is not present"); + + Path jdk8 = findJdk8(); + BackendTestSupport.require(jdk8 != null, "no JDK 8 available to compile the backend"); + + work = Files.createTempDirectory("backend-http-test"); + Path binary = work.resolve("petserver"); + Path staticRoot = Files.createDirectories(work.resolve("www")); + Files.write(staticRoot.resolve("index.html"), + "

index

".getBytes(StandardCharsets.UTF_8)); + byte[] blob = new byte[256 * 1024]; + for (int i = 0; i < blob.length; i++) { + blob[i] = (byte) (i & 0xff); + } + Files.write(staticRoot.resolve("big.bin"), blob); + // Deliberately larger than any socket send buffer: a slow reader has to + // make the server's write block partway through, which is the only way + // to reach the backpressure paths in send() and sendfile(). + byte[] huge = new byte[HUGE_BYTES]; + for (int i = 0; i < huge.length; i++) { + huge[i] = (byte) ((i * 31) & 0xff); + } + Files.write(staticRoot.resolve("huge.bin"), huge); + + // The real build script, not a reimplementation of it: a test that builds + // differently from the product is testing something else. + ProcessBuilder build = new ProcessBuilder("./build.sh", "PetServer", "com.demo", + binary.toString()); + build.directory(backend.toFile()); + build.environment().put("JDK_8_HOME", jdk8.toString()); + build.environment().put("JAVA_HOME", jdk8.toString()); + build.environment().put("CN1_BACKEND_DEMO", "demo/petserver"); + build.redirectErrorStream(true); + Process p = build.start(); + String buildLog = readFully(p.getInputStream()); + boolean built = p.waitFor(20, TimeUnit.MINUTES) && p.exitValue() == 0 + && Files.isExecutable(binary); + if (!built) { + String tail = buildLog.length() > 3000 + ? buildLog.substring(buildLog.length() - 3000) : buildLog; + BackendTestSupport.skipOrFail("could not build the backend binary:\n" + tail); + } + + port = freePort(); + ProcessBuilder run = new ProcessBuilder(binary.toString()); + run.environment().put("CN1_PORT", String.valueOf(port)); + run.environment().put("CN1_DB_PATH", work.resolve("test.db").toString()); + run.environment().put("CN1_STATIC_ROOT", staticRoot.toString()); + run.environment().put("CN1_HTTP_TIMEOUT_MS", "4000"); + run.redirectErrorStream(true); + run.redirectOutput(work.resolve("server.log").toFile()); + server = run.start(); + assertTrue(waitForPort(port, 30000), "the server never accepted a connection"); + + startTlsServer(work, binary, staticRoot); + startBusyServer(work, binary, staticRoot); + startSmallUploadServer(work, binary, staticRoot); + } + + /** + * A copy with a SMALL in-flight upload budget, so the budget can be reached + * with megabytes instead of the default sixty-four. A test that has to move + * 64MB to reach a limit is a test nobody runs. + */ + private static void startSmallUploadServer(Path work, Path binary, Path staticRoot) + throws Exception { + smallUploadPort = freePort(); + ProcessBuilder run = new ProcessBuilder(binary.toString()); + run.environment().put("CN1_PORT", String.valueOf(smallUploadPort)); + run.environment().put("CN1_DB_PATH", work.resolve("upload.db").toString()); + run.environment().put("CN1_STATIC_ROOT", staticRoot.toString()); + run.environment().put("CN1_HTTP_MAX_UPLOAD_MB", "16"); + // And a small HTTP/2 body ceiling, so that one can be reached with a + // few megabytes as well. + run.environment().put("CN1_HTTP_MAX_H2_BODY_MB", "4"); + // A DELIBERATELY invalid rate. Zero is a divisor in fillTo() and in + // requireChunkedProgress(), so an unguarded server throws + // ArithmeticException on the first body needing a second read and + // drops the connection with no response at all. Set here rather than + // on its own fixture so every upload test on this port carries the + // proof, and named in a test below so it cannot be deleted as noise. + run.environment().put("CN1_HTTP_MIN_BODY_RATE", "0"); + run.redirectErrorStream(true); + smallUploadLog = work.resolve("upload-server.log"); + run.redirectOutput(smallUploadLog.toFile()); + smallUploadServer = run.start(); + if (!waitForPort(smallUploadPort, 30000)) { + smallUploadServer.destroy(); + smallUploadServer = null; + smallUploadPort = 0; + } + } + + /** The single-host server described on busyServer. */ + private static void startBusyServer(Path work, Path binary, Path staticRoot) + throws Exception { + busyPort = freePort(); + ProcessBuilder run = new ProcessBuilder(binary.toString()); + run.environment().put("CN1_PORT", String.valueOf(busyPort)); + run.environment().put("CN1_DB_PATH", work.resolve("busy.db").toString()); + run.environment().put("CN1_STATIC_ROOT", staticRoot.toString()); + run.environment().put("CN1_HTTP_TIMEOUT_MS", "2000"); + run.environment().put("CN1_WORKERS", "1"); + run.redirectErrorStream(true); + run.redirectOutput(work.resolve("busy-server.log").toFile()); + busyServer = run.start(); + if (!waitForPort(busyPort, 30000)) { + busyServer.destroy(); + busyServer = null; + busyPort = 0; + } + } + + /** + * Starts a second copy of the same binary with a certificate configured. + * + * TLS had no end-to-end coverage at all: every other test here speaks + * plaintext, so the handshake, the record layer and the ALPN negotiation + * were exercised by nothing. Reusing the binary just built keeps that to one + * extra process rather than a second translation. + */ + private void startTlsServer(Path work, Path binary, Path staticRoot) throws Exception { + Path cert = work.resolve("cert.pem"); + Path key = work.resolve("key.pem"); + ProcessBuilder openssl = new ProcessBuilder("openssl", "req", "-x509", "-newkey", + "rsa:2048", "-keyout", key.toString(), "-out", cert.toString(), + "-days", "1", "-nodes", "-subj", "/CN=localhost"); + openssl.redirectErrorStream(true); + openssl.redirectOutput(work.resolve("openssl.log").toFile()); + Process made; + try { + made = openssl.start(); + } catch (IOException noOpenssl) { + // Without a certificate there is nothing to serve; the plaintext + // tests still run and the TLS ones report why they did not. + return; + } + if (!made.waitFor(60, TimeUnit.SECONDS) || made.exitValue() != 0 + || !Files.exists(cert) || !Files.exists(key)) { + return; + } + tlsPort = freePort(); + ProcessBuilder run = new ProcessBuilder(binary.toString()); + run.environment().put("CN1_PORT", String.valueOf(tlsPort)); + run.environment().put("CN1_DB_PATH", work.resolve("tls.db").toString()); + run.environment().put("CN1_STATIC_ROOT", staticRoot.toString()); + run.environment().put("CN1_HTTP_TIMEOUT_MS", "4000"); + run.environment().put("CN1_TLS_CERT", cert.toString()); + run.environment().put("CN1_TLS_KEY", key.toString()); + run.redirectErrorStream(true); + run.redirectOutput(work.resolve("tls-server.log").toFile()); + tlsServer = run.start(); + if (!waitForPort(tlsPort, 30000)) { + tlsServer.destroyForcibly(); + tlsServer = null; + tlsPort = 0; + } + } + + @AfterAll + void stopServer() { + if (smallUploadServer != null) { + smallUploadServer.destroy(); + try { + if (!smallUploadServer.waitFor(10, TimeUnit.SECONDS)) { + smallUploadServer.destroyForcibly(); + } + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + } + if (busyServer != null) { + busyServer.destroy(); + try { + if (!busyServer.waitFor(10, TimeUnit.SECONDS)) { + busyServer.destroyForcibly(); + } + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + } + if (tlsServer != null) { + tlsServer.destroy(); + try { + if (!tlsServer.waitFor(10, TimeUnit.SECONDS)) { + tlsServer.destroyForcibly(); + } + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + } + if (server != null) { + server.destroy(); + try { + if (!server.waitFor(10, TimeUnit.SECONDS)) { + server.destroyForcibly(); + } + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + } + } + + // ------------------------------------------------------------------ + // Functional + // ------------------------------------------------------------------ + + @Test + @DisplayName("a DTO round-trips through the generated dispatcher and SQLite") + void dtoRoundTrip() throws Exception { + String created = body(request("POST", "/pet", + "{\"name\":\"Fido\",\"species\":\"dog\",\"weight\":12.5,\"good\":true}", null)); + assertTrue(created.contains("\"name\":\"Fido\""), created); + assertTrue(created.contains("\"weight\":12.5"), "a double must survive the round trip: " + created); + assertTrue(created.contains("\"good\":true"), "a boolean must survive the round trip: " + created); + + String listed = body(request("GET", "/pets", null, null)); + assertTrue(listed.startsWith("["), listed); + assertTrue(listed.contains("Fido"), listed); + } + + @Test + @DisplayName("headers and cookies bind from the request") + void headersAndCookiesBind() throws Exception { + String response = body(request("GET", "/whoami", null, + new String[]{"X-User: shai", "Cookie: theme=dark; session=abc123"})); + assertTrue(response.contains("user=shai"), response); + assertTrue(response.contains("session=abc123"), response); + } + + @Test + @DisplayName("a protected route needs a valid bearer token") + void authGuardsMutatingRoutes() throws Exception { + assertEquals(401, status(request("DELETE", "/pet/9999", null, null))); + + String token = body(request("POST", "/login", + "{\"username\":\"shai\",\"password\":\"hunter2\"}", null)).replace("\"", ""); + assertTrue(token.split("\\.").length == 3, "expected a three-part JWT, got: " + token); + + assertEquals(401, status(request("POST", "/login", + "{\"username\":\"shai\",\"password\":\"wrong\"}", null))); + // An unknown user and a wrong password must be indistinguishable, or the + // login endpoint becomes a list of valid usernames. + assertEquals(401, status(request("POST", "/login", + "{\"username\":\"nobody\",\"password\":\"hunter2\"}", null))); + + String created = body(request("POST", "/pet", "{\"name\":\"Doomed\"}", null)); + // Asserted rather than substring'd straight away: indexOf returning -1 here + // throws StringIndexOutOfBounds and takes the response with it, which is + // how an intermittent malformed body was reported for a while as nothing + // more than "String index out of range: -1". + // + // KNOWN OPEN DEFECT, and this assertion is what finally named it: the + // server occasionally answers a request with NOTHING. The body captured + // here came back empty, and transactionRollsBack fails the same way from + // the other side -- status -1 after exactly 15.05s, which is this class's + // own setSoTimeout(15000) expiring with no reply. Measured on the + // dispatching path at 2 failures in 4 full-suite runs (about 100 requests + // each), and it predates the virtual-thread work: a build with none of it + // fails identically. It does NOT reproduce in isolation -- 400 plain + // POSTs and 120 replays of this test's exact request sequence were both + // clean -- so the trigger is interaction with the other tests against the + // shared server, not this request. Not a flake to be re-run: a request + // that goes unanswered is a server bug and this is where it surfaced. + assertTrue(created.indexOf(":") >= 0 && created.indexOf(",") >= 0, + "POST /pet returned a body that is not the expected JSON: [" + created + "]"); + String id = created.substring(created.indexOf(":") + 1, created.indexOf(",")); + assertEquals(200, status(request("DELETE", "/pet/" + id, null, + new String[]{"Authorization: Bearer " + token}))); + + String tampered = token.substring(0, token.length() - 1) + "X"; + assertEquals(401, status(request("DELETE", "/pet/1", null, + new String[]{"Authorization: Bearer " + tampered}))); + } + + @Test + @DisplayName("a failed batch rolls back every row it had already written") + void transactionRollsBack() throws Exception { + String token = body(request("POST", "/login", + "{\"username\":\"shai\",\"password\":\"hunter2\"}", null)).replace("\"", ""); + int before = countPets(); + assertEquals(400, status(request("POST", "/pets/bulk", + "[{\"name\":\"Rollback1\"},{\"species\":\"nameless\"}]", + new String[]{"Authorization: Bearer " + token}))); + assertEquals(before, countPets(), + "the row written before the failure must not survive"); + } + + @Test + @DisplayName("a chunked body is decoded") + void chunkedUpload() throws Exception { + String payload = "{\"name\":\"Chunky\",\"species\":\"cat\"}"; + StringBuilder chunks = new StringBuilder(); + for (int i = 0; i < payload.length(); i += 7) { + String part = payload.substring(i, Math.min(i + 7, payload.length())); + chunks.append(Integer.toHexString(part.length())).append("\r\n").append(part).append("\r\n"); + } + chunks.append("0\r\n\r\n"); + byte[] response = raw("POST /pet HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\n" + + "Connection: close\r\n\r\n" + chunks); + assertEquals(200, statusOf(response), new String(response, StandardCharsets.UTF_8)); + assertTrue(new String(response, StandardCharsets.UTF_8).contains("Chunky")); + } + + @Test + @DisplayName("metrics report what the server is doing") + void metricsEndpoint() throws Exception { + String health = body(request("GET", "/healthz", null, null)); + assertTrue(health.contains("\"status\":\"ok\""), health); + assertTrue(health.contains("\"requestsServed\""), health); + assertTrue(health.contains("\"activeRequests\""), health); + } + + // ------------------------------------------------------------------ + // Static files + // ------------------------------------------------------------------ + + @Test + @DisplayName("static files serve, 404, and refuse to leave the document root") + void staticFileBasics() throws Exception { + assertTrue(body(request("GET", "/static/index.html", null, null)).contains("

index

")); + assertEquals(404, status(request("GET", "/static/missing.html", null, null))); + // Percent-encoded traversal: a check on the raw request string misses this. + int traversal = status(request("GET", "/static/..%2f..%2fetc%2fpasswd", null, null)); + assertTrue(traversal == 403 || traversal == 404, + "a traversal must not be served, got " + traversal); + } + + @Test + @DisplayName("conditional requests answer 304") + void conditionalGet() throws Exception { + byte[] first = request("GET", "/static/index.html", null, null); + String etag = header(first, "ETag"); + assertNotNull(etag, "a static response must carry an ETag"); + // And the 304 must not claim a length. Content-Length on a 304 describes + // the SELECTED REPRESENTATION -- what a 200 would have sent -- and the + // only figure available here is the empty body's zero, which would tell + // the cache the file it just validated is empty. The header is optional + // on a 304, so it is omitted rather than fabricated. + byte[] conditional = raw("GET /static/index.html HTTP/1.1\r\nHost: x\r\n" + + "If-None-Match: " + etag + "\r\nConnection: close\r\n\r\n"); + String conditionalText = new String(conditional, StandardCharsets.UTF_8); + assertTrue(conditionalText.startsWith("HTTP/1.1 304"), + "a matching ETag should answer 304:\n" + conditionalText); + assertEquals(-1, conditionalText.toLowerCase().indexOf("content-length"), + "a 304 must not advertise a length it cannot describe:\n" + conditionalText); + assertEquals(304, status(request("GET", "/static/index.html", null, + new String[]{"If-None-Match: " + etag}))); + + String lastModified = header(first, "Last-Modified"); + assertNotNull(lastModified, "a static response must carry Last-Modified"); + assertEquals(304, status(request("GET", "/static/index.html", null, + new String[]{"If-Modified-Since: " + lastModified}))); + } + + @Test + @DisplayName("ranges are honoured and an impossible one is refused") + void rangeRequests() throws Exception { + byte[] partial = request("GET", "/static/big.bin", null, + new String[]{"Range: bytes=0-99"}); + assertEquals(206, statusOf(partial)); + assertEquals("bytes 0-99/262144", header(partial, "Content-Range")); + assertEquals("100", header(partial, "Content-Length")); + + assertEquals(416, status(request("GET", "/static/big.bin", null, + new String[]{"Range: bytes=999999999-"}))); + + // A multi-range request is VALID and satisfiable; this server just does + // not assemble multipart/byteranges. 416 asserts that none of what was + // asked for exists, which is a different and untrue statement, so the + // Range is ignored and the whole representation is sent instead. + byte[] multi = request("GET", "/static/big.bin", null, + new String[]{"Range: bytes=0-99,200-299"}); + assertEquals(200, statusOf(multi), + "a satisfiable multi-range must not be refused as unsatisfiable"); + assertEquals("262144", header(multi, "Content-Length")); + assertEquals(null, header(multi, "Content-Range"), + "a 200 describes the whole representation, so it carries no Content-Range"); + + // And a Range that cannot be parsed at all is ignored for the same reason. + assertEquals(200, status(request("GET", "/static/big.bin", null, + new String[]{"Range: bytes=abc"}))); + } + + @Test + @DisplayName("HEAD reports the length a GET would send, not zero") + void headReportsRealLength() throws Exception { + byte[] head = request("HEAD", "/static/big.bin", null, null); + assertEquals(200, statusOf(head)); + assertEquals("262144", header(head, "Content-Length"), + "a HEAD that reports 0 tells the client the resource is empty"); + assertEquals(0, bodyBytes(head).length, "a HEAD response must carry no body"); + } + + // ------------------------------------------------------------------ + // Protocol correctness. No ordinary client sends any of this, which is + // exactly why a server has to get it right. + // ------------------------------------------------------------------ + + @Test + @DisplayName("two pipelined requests both get answered") + void pipelinedRequestsAreNotLost() throws Exception { + byte[] response = raw("GET /healthz HTTP/1.1\r\nHost: x\r\n\r\n" + + "GET /healthz HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + int responses = countOccurrences(new String(response, StandardCharsets.UTF_8), "HTTP/1.1 "); + assertEquals(2, responses, + "a client may send a second request before reading the first reply"); + } + + @Test + @DisplayName("a body on a 204 is suppressed rather than desynchronising the connection") + void bodilessStatusDoesNotDesyncTheConnection() throws Exception { + // /nocontent returns a 204 WITH bytes, which a handler is free to build. + // RFC 9110 ends such a response at the header section, so writing them + // would leave the client reading "junk" as the start of the second reply + // and everything after that misframed. Both requests go out together so + // that a desync is visible as a wrong reply rather than a slow one. + byte[] response = raw("GET /nocontent HTTP/1.1\r\nHost: x\r\n\r\n" + + "GET /healthz HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + String text = new String(response, StandardCharsets.UTF_8); + assertTrue(text.startsWith("HTTP/1.1 204"), "the first reply should be a 204:\n" + text); + assertEquals(-1, text.indexOf("junk"), + "a 204 must not carry a body:\n" + text); + assertEquals(2, countOccurrences(text, "HTTP/1.1 "), + "both replies must be readable back to back:\n" + text); + // RFC 9110 6.4.1 makes Content-Length a MUST NOT on a 204, and sending one + // is its own desync: a keep-alive client would wait for bytes never sent. + String head = text.substring(0, text.indexOf("\r\n\r\n") + 4); + assertEquals(-1, head.toLowerCase().indexOf("content-length"), + "a 204 must not carry Content-Length:\n" + head); + } + + @Test + @DisplayName("a percent-encoded non-ASCII parameter name is found") + void nonAsciiQueryNamesMatchTheirUtf8Encoding() throws Exception { + // caf%C3%A9 is how every client sends this name. Decoded it is the two + // octets 0xC3 0xA9, and the Java char is 0xE9 -- so comparing an octet + // to a char can never match, and the parameter reads as absent with the + // handler quietly using its default instead. + byte[] response = raw("GET /accent?caf%C3%A9=au-lait HTTP/1.1\r\nHost: x\r\n" + + "Connection: close\r\n\r\n"); + String text = new String(response, StandardCharsets.UTF_8); + assertTrue(text.indexOf("au-lait") >= 0, + "the encoded name must match the declared one:\n" + text); + } + + @Test + @DisplayName("a silent connection is shed even while its host stays busy") + void deadlinesAreSweptOnABusyHost() throws Exception { + // The sweep used to run only when a poll came back EMPTY, so a host that + // always had an event never swept -- and a client can keep that true with + // a trickle of traffic while its other connections sit silent, holding + // them past any timeout until the process ceiling is reached. + // + // This server is pinned to one virtual-thread host (CN1_WORKERS=1), so + // the traffic below and the silent connection are certainly on the same + // one. With several hosts they might not be, and the test would pass by + // luck rather than by the fix. + Assumptions.assumeTrue(busyServer != null && busyPort != 0, + "the single-host server is not running"); + Socket quiet = new Socket(); + quiet.connect(new InetSocketAddress("127.0.0.1", busyPort), 5000); + quiet.setSoTimeout(12000); + try { + // Never speaks. Its deadline is the only thing that can close it. + InputStream in = quiet.getInputStream(); + long deadline = System.currentTimeMillis() + 10000; + boolean closed = false; + while (System.currentTimeMillis() < deadline) { + // Keep the host receiving events, so a poll never comes back empty. + Socket chatter = new Socket(); + chatter.connect(new InetSocketAddress("127.0.0.1", busyPort), 5000); + chatter.setSoTimeout(5000); + try { + chatter.getOutputStream().write(("GET /healthz HTTP/1.1\r\nHost: x\r\n" + + "Connection: close\r\n\r\n").getBytes(StandardCharsets.UTF_8)); + chatter.getOutputStream().flush(); + while (chatter.getInputStream().read() >= 0) { + // drain + } + } finally { + chatter.close(); + } + if (in.available() > 0 || quiet.isClosed()) { + closed = true; + break; + } + // A read with a short timeout tells us whether the peer hung up. + quiet.setSoTimeout(200); + try { + if (in.read() < 0) { + closed = true; + break; + } + } catch (java.net.SocketTimeoutException stillOpen) { + // expected while the deadline has not yet passed + } + } + assertTrue(closed, "a connection that never spoke must be shed by its " + + "deadline even while the host is busy"); + } finally { + quiet.close(); + } + } + + @Test + @DisplayName("the packaged client talks to the packaged server") + void theTranslatedClientDrivesTheServer() throws Exception { + // ParparVM on BOTH ends. Every other test here drives the server from + // JUnit, over a raw socket or an HttpURLConnection, so the packaged + // OUTBOUND client -- Web, and the libcurl under it -- was never exercised + // against a real server at all. Two things only this can show: that each + // verb arrives as itself, PATCH included, which the Java SE arm cannot + // send and therefore cannot test; and that the two halves agree when they + // actually meet. + Path work = Files.createTempDirectory("backend-webcheck"); + Path clientBinary = work.resolve("webcheck"); + Path jdk8 = BackendTestSupport.findJdk8(); + BackendTestSupport.require(jdk8 != null, "no JDK 8 available to build the client"); + String failure = BackendTestSupport.build("WebCheck", "demo/webcheck", clientBinary, jdk8); + if (failure != null) { + BackendTestSupport.skipOrFail(failure); + return; + } + ProcessBuilder run = new ProcessBuilder(clientBinary.toString()); + run.environment().put("CN1_WEBCHECK_BASE", "http://127.0.0.1:" + port); + run.redirectErrorStream(true); + Process client = run.start(); + ByteArrayOutputStream captured = new ByteArrayOutputStream(); + InputStream clientOut = client.getInputStream(); + byte[] chunk = new byte[4096]; + int n; + while ((n = clientOut.read(chunk)) > 0) { + captured.write(chunk, 0, n); + } + String out = new String(captured.toByteArray(), StandardCharsets.UTF_8); + int exit = client.waitFor(); + assertTrue(out.contains("WEBCHECK OK"), + "the translated client reported failures against the server:\n" + out); + assertEquals(0, exit, "the translated client exited nonzero:\n" + out); + } + + @Test + @DisplayName("a megabyte-scale upload is read whole and answered") + void aLargeUploadIsReadWhole() throws Exception { + // The whole upload path had no test with a body big enough to grow the + // buffer more than once: the 8MB fixture is a static FILE, so it exercises + // downloads. That left the doubling growth, the rate bound and the + // in-flight budget all resting on small bodies. Two megabytes crosses the + // starting chunk about seven times. + StringBuilder json = new StringBuilder(2 * 1024 * 1024 + 16); + json.append("[\""); + for (int i = 0; i < 2 * 1024 * 1024; i++) { + json.append('a'); + } + json.append("\"]"); + byte[] body = json.toString().getBytes(StandardCharsets.UTF_8); + byte[] response = raw("POST /api/notes HTTP/1.1\r\nHost: x\r\nContent-Type: " + + "application/json\r\nContent-Length: " + body.length + + "\r\nConnection: close\r\n\r\n", body); + String text = new String(response, StandardCharsets.UTF_8); + assertTrue(text.startsWith("HTTP/1.1 "), + "a large upload must be answered, not dropped:\n" + + text.substring(0, Math.min(200, text.length()))); + assertEquals(-1, text.substring(0, Math.min(64, text.length())).indexOf(" 503"), + "a legitimate upload must not hit the in-flight budget:\n" + + text.substring(0, Math.min(200, text.length()))); + } + + @Test + @DisplayName("a chunk arriving after a pause is not mistaken for a hangup") + void aChunkInASecondPacketIsNotAHangup() throws Exception { + // Plaintext descriptors are NON-BLOCKING in virtual-thread mode, so a read + // with nothing ready gets EAGAIN, and reporting that as end of stream drops + // a request that was still arriving. + // + // It has to be CHUNKED to reach that read. A Content-Length body goes + // through fillTo(), which uses the copying path and parks correctly -- a + // first version of this test used one, passed with the fix reverted, and + // proved nothing. readChunked calls fill(), which takes the zero-copy + // branch once the buffered bytes run out, and that is the read in + // question. + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(20000); + try { + OutputStream out = socket.getOutputStream(); + // Headers and the first chunk together, so the head is fully parsed and + // the buffered bytes are consumed before the gap. + out.write(("POST /echo HTTP/1.1\r\nHost: x\r\nContent-Type: application/json\r\n" + + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + + "2\r\n[\"\r\n").getBytes(StandardCharsets.UTF_8)); + out.flush(); + Thread.sleep(400); + try { + out.write("5\r\nsplit\r\n".getBytes(StandardCharsets.UTF_8)); + out.flush(); + Thread.sleep(400); + out.write("2\r\n\"]\r\n0\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + out.flush(); + } catch (IOException closedDuringTheGap) { + // Said in words rather than as a raw socket error: when this + // regresses, the server has hung up mid-upload and the next write + // meets a closed socket. "Broken pipe" alone does not say that. + fail("the server closed the connection while the body was still " + + "arriving, so a valid chunked upload was dropped: " + + closedDuringTheGap); + } + byte[] response = readFullyBytes(socket.getInputStream()); + assertEquals(200, status(response), + "the chunks arrived in separate packets and the request was dropped:\n" + + new String(response, StandardCharsets.UTF_8)); + String text = new String(response, StandardCharsets.UTF_8); + assertTrue(text.indexOf("len=9") > 0, + "every chunk must reach the handler, got:\n" + text); + } finally { + socket.close(); + } + } + + @Test + @DisplayName("an invalid minimum rate falls back instead of dividing by zero") + void aZeroMinimumBodyRateDoesNotKillTheConnection() throws Exception { + // CN1_HTTP_MIN_BODY_RATE=0 reaches a division in both body readers. The + // ArithmeticException that follows is caught as an ordinary read failure, + // so the connection is dropped WITHOUT a response -- a setting that looks + // like a tuning knob and silently makes every upload fail. + // + // This fixture server runs with that value on purpose, so the upload tests + // above already depend on the fallback; this one says so out loud. + Assumptions.assumeTrue(smallUploadPort > 0, + "the small-budget server did not start"); + // The clamp SAYS SO on stderr, and the fixture's output is captured, so + // this is what actually bites when the guard is removed: the two runtimes + // disagree about what a zero divisor does -- Java SE throws + // ArithmeticException and drops the connection, ParparVM answers 0 and + // quietly loses the rate part of the deadline -- so behaviour alone cannot + // catch it on both. The message can. + String log = new String(java.nio.file.Files.readAllBytes(smallUploadLog), + StandardCharsets.UTF_8); + // JUnit 5 here: condition first, message second. + assertTrue(log.indexOf("CN1_HTTP_MIN_BODY_RATE=0 is below the minimum") >= 0, + "the server did not report refusing CN1_HTTP_MIN_BODY_RATE=0:\n" + log); + + // And it still serves. The body has to arrive in a SECOND packet: sent in + // one write it is already buffered when fillTo() looks, so the method + // returns before it ever computes the allowance -- a first version of this + // test did exactly that and passed with the guard removed. + byte[] body = ("[\"" + repeat('a', 4096) + "\"]").getBytes(StandardCharsets.UTF_8); + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", smallUploadPort), 5000); + socket.setSoTimeout(20000); + try { + OutputStream out = socket.getOutputStream(); + out.write(("POST /echo HTTP/1.1\r\nHost: x\r\nContent-Type: application/json\r\n" + + "Content-Length: " + body.length + "\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.UTF_8)); + out.flush(); + Thread.sleep(300); + out.write(body); + out.flush(); + byte[] response = readFullyBytes(socket.getInputStream()); + assertEquals(200, status(response), + "a body needing a second read must still be served:\n" + + new String(response, StandardCharsets.UTF_8)); + } finally { + socket.close(); + } + } + + private static String repeat(char c, int count) { + StringBuilder sb = new StringBuilder(count); + for (int i = 0; i < count; i++) { + sb.append(c); + } + return sb.toString(); + } + + @Test + @DisplayName("chunked uploads are charged against the process budget too") + void chunkedUploadsAreChargedAndReleased() throws Exception { + // The budget bounded the fixed-length reader and nothing else, so a + // chunked body was capped per request at 8MB and by nothing at all across + // requests: enough clients sending almost that much and pausing before the + // terminating chunk retain gigabytes with CN1_HTTP_MAX_UPLOAD_MB looking + // on. + // + // Sized so that a leak is what fails: nine 2MB bodies against the 16MB + // ceiling charge 18MB cumulatively, while each one alone peaks at 2MB. If + // the charge were never made the release could not leak either, so this + // proves both halves are wired -- and it is sequential on purpose, because + // detecting the leak needs accumulation, not concurrency. + Assumptions.assumeTrue(smallUploadPort > 0, + "the small-budget server did not start"); + for (int i = 0; i < 9; i++) { + byte[] response = chunkedPost(smallUploadPort, 2 * 1024 * 1024); + assertEquals(200, status(response), + "chunked upload " + i + " was refused, so an earlier one's " + + "reservation was never released:\n" + + new String(response, StandardCharsets.UTF_8)); + } + } + + /** + * Posts `size` bytes of JSON to /echo, chunk-encoded. + * + * Built whole and written in one go rather than streamed: writing it + * incrementally raced the server's own answer, so a legitimate early response + * arrived as a broken pipe on the next write and the status that explained it + * was never read. + */ + private byte[] chunkedPost(int onPort, int size) throws IOException { + ByteArrayOutputStream framed = new ByteArrayOutputStream(); + framed.write("2\r\n[\"\r\n".getBytes(StandardCharsets.UTF_8)); + byte[] payload = new byte[256 * 1024]; + java.util.Arrays.fill(payload, (byte) 'a'); + int sent = 0; + while (sent < size) { + int n = Math.min(payload.length, size - sent); + framed.write((Integer.toHexString(n) + "\r\n").getBytes(StandardCharsets.UTF_8)); + framed.write(payload, 0, n); + framed.write("\r\n".getBytes(StandardCharsets.UTF_8)); + sent += n; + } + framed.write("2\r\n\"]\r\n".getBytes(StandardCharsets.UTF_8)); + framed.write("0\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + return rawOn(onPort, "POST /echo HTTP/1.1\r\nHost: x\r\n" + + "Content-Type: application/json\r\n" + + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + framed.toByteArray()); + } + + @Test + @DisplayName("a body that is not UTF-8 is refused rather than repaired") + void malformedUtf8BodiesAreRefused() throws Exception { + // new String(bytes, "UTF-8") never fails: it substitutes U+FFFD, so the + // handler ran on text the client never sent and anything that validated + // the body validated the REPLACEMENT. 0x80 is a continuation byte with + // nothing to continue, inside an otherwise perfectly good JSON string. + byte[] body = new byte[] { + '[', '"', 'a', (byte) 0x80, 'b', '"', ']', + }; + byte[] response = raw("POST /echo HTTP/1.1\r\nHost: x\r\nContent-Type: " + + "application/json\r\nContent-Length: " + body.length + + "\r\nConnection: close\r\n\r\n", body); + assertEquals(400, status(response), + "a malformed sequence must be a 400, not a silent replacement:\n" + + new String(response, StandardCharsets.UTF_8)); + + // Multi-byte UTF-8 that IS well formed still has to get through -- the + // rule is about malformed bytes, not about non-ASCII. + byte[] good = ("[\"caf\u00e9\"]").getBytes(StandardCharsets.UTF_8); + byte[] ok = raw("POST /echo HTTP/1.1\r\nHost: x\r\nContent-Type: " + + "application/json\r\nContent-Length: " + good.length + + "\r\nConnection: close\r\n\r\n", good); + assertEquals(200, status(ok), new String(ok, StandardCharsets.UTF_8)); + } + + @Test + @DisplayName("a chunked body that is not UTF-8 is refused too") + void malformedUtf8ChunkedBodiesAreRefused() throws Exception { + // The chunked path decodes separately, so it needs its own proof: fixing + // one of two body readers is how the fixed-length path came to be bounded + // while this one was not. + String chunk = "5\r\n"; + byte[] head = ("POST /echo HTTP/1.1\r\nHost: x\r\nContent-Type: application/json\r\n" + + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + chunk) + .getBytes(StandardCharsets.UTF_8); + byte[] payload = new byte[] { '[', '"', (byte) 0xC3, '"', ']' }; + byte[] tail = "\r\n0\r\n\r\n".getBytes(StandardCharsets.UTF_8); + byte[] all = new byte[head.length + payload.length + tail.length]; + System.arraycopy(head, 0, all, 0, head.length); + System.arraycopy(payload, 0, all, head.length, payload.length); + System.arraycopy(tail, 0, all, head.length + payload.length, tail.length); + byte[] response = rawBytes(all); + assertEquals(400, status(response), + "a truncated multi-byte sequence must be a 400:\n" + + new String(response, StandardCharsets.UTF_8)); + } + + @Test + @DisplayName("concurrent uploads reserve and release their budget") + void concurrentUploadsDoNotLeakTheirBudget() throws Exception { + // The in-flight budget is what bounds concurrent uploads, so it is charged + // BEFORE the memory is allocated -- a budget checked afterwards bounds + // nothing, since every thread at a growth boundary takes its memory first + // and learns it was over the limit second. + // + // The ORDER is not observable from out here. A leaked RESERVATION is, and + // only if the numbers are chosen for it: against a 16MB budget, three + // concurrent 2MB uploads peak at 6MB and pass, while three rounds of them + // charge 18MB cumulatively and start answering 503 the moment the release + // stops happening. A first version of this test ran four rounds of six + // against the DEFAULT 64MB budget -- 48MB, which never reaches the limit, + // so it passed with the release deleted and proved nothing. + Assumptions.assumeTrue(smallUploadPort > 0, + "the small-upload-budget server did not start"); + final int rounds = 3; + final int concurrent = 3; + StringBuilder json = new StringBuilder(2 * 1024 * 1024 + 16); + json.append("[\""); + for (int i = 0; i < 2 * 1024 * 1024; i++) { + json.append('a'); + } + json.append("\"]"); + final byte[] body = json.toString().getBytes(StandardCharsets.UTF_8); + + for (int round = 0; round < rounds; round++) { + final String[] outcomes = new String[concurrent]; + Thread[] threads = new Thread[concurrent]; + for (int i = 0; i < concurrent; i++) { + final int slot = i; + threads[i] = new Thread(new Runnable() { + public void run() { + try { + byte[] response = rawOn(smallUploadPort, + "POST /api/notes HTTP/1.1\r\nHost: x\r\n" + + "Content-Type: application/json\r\nContent-Length: " + + body.length + "\r\nConnection: close\r\n\r\n", body); + String text = new String(response, StandardCharsets.UTF_8); + outcomes[slot] = text.substring(0, Math.min(32, text.length())); + } catch (Exception err) { + outcomes[slot] = "threw: " + err; + } + } + }); + threads[i].start(); + } + for (int i = 0; i < concurrent; i++) { + threads[i].join(120000); + } + for (int i = 0; i < concurrent; i++) { + assertNotNull(outcomes[i], "upload " + i + " of round " + round + + " never answered"); + assertEquals(-1, outcomes[i].indexOf(" 503"), + "round " + round + " upload " + i + " hit the in-flight budget, so a " + + "reservation from an earlier round was never released: " + + outcomes[i]); + } + } + } + + @Test + @DisplayName("deeply nested JSON is refused without taking the server down") + void deeplyNestedJsonDoesNotOverflowTheStack() throws Exception { + // Handlers run on a 64KB virtual-thread stack and the JSON parser is + // recursive, so nesting depth IS stack depth. A depth just UNDER the + // parser's own cap is the dangerous one: the cap lets it through and + // the stack decides what happens next. That is a kilobyte of body from + // an unauthenticated client, and a StackOverflowError is an Error -- + // no handler catch and no server catch of Exception sees it. + StringBuilder deep = new StringBuilder(); + int depth = 511; + for (int i = 0; i < depth; i++) { + deep.append('['); + } + for (int i = 0; i < depth; i++) { + deep.append(']'); + } + byte[] body = deep.toString().getBytes(StandardCharsets.UTF_8); + byte[] response = raw("POST /api/notes HTTP/1.1\r\nHost: x\r\nContent-Type: " + + "application/json\r\nContent-Length: " + body.length + + "\r\nConnection: close\r\n\r\n", body); + String text = new String(response, StandardCharsets.UTF_8); + // The status matters: it proves the body was PARSED rather than rejected + // before the parser ever recursed, which would make this test vacuous. + // Under the cap the document is valid, so the route answers as it would + // for any other body -- what must not happen is silence or a dead server. + String head = text.substring(0, Math.max(0, text.indexOf("\r\n"))); + assertTrue(text.startsWith("HTTP/1.1 "), + "a nested body must be answered, not dropped:\n" + text); + assertEquals(-1, head.indexOf(" 500"), + "a legal document under the parser's own cap must not fault:\n" + head); + // And the server has to still be there afterwards -- a crash shows up + // here rather than in the reply above. + byte[] after = raw("GET /healthz HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + String health = new String(after, StandardCharsets.UTF_8); + assertTrue(health.startsWith("HTTP/1.1 200"), + "the server must survive a deeply nested body:\n" + health); + } + + @Test + @DisplayName("a bare LF inside a header value is refused, not carried") + void headerValuesMayNotHideAnotherField() throws Exception { + // This parser ends a field at CRLF, so a bare LF in a value is just a + // byte to it -- while an intermediary that accepts bare LF as a + // delimiter reads TWO fields here, the second a Content-Length, and + // frames the body by it. One connection read two ways is how the next + // request on it becomes whatever the attacker appended. + byte[] smuggled = raw("GET /healthz HTTP/1.1\r\nHost: x\r\n" + + "X-Thing: value\nContent-Length: 5\r\nConnection: close\r\n\r\n"); + String text = new String(smuggled, StandardCharsets.UTF_8); + assertTrue(text.startsWith("HTTP/1.1 400"), + "a header value carrying a bare LF must be refused:\n" + text); + + // A name that is not a token goes the same way. + byte[] badName = raw("GET /healthz HTTP/1.1\r\nHost: x\r\n" + + "X Thing: value\r\nConnection: close\r\n\r\n"); + assertTrue(new String(badName, StandardCharsets.UTF_8).startsWith("HTTP/1.1 400"), + "a field name that is not a token must be refused"); + + // And an ordinary request still works, including a tab inside a value, + // which RFC 9110 allows and which a blanket control-character rule would + // have broken. + byte[] ok = raw("GET /healthz HTTP/1.1\r\nHost: x\r\nX-Thing: a\tb\r\n" + + "Connection: close\r\n\r\n"); + assertTrue(new String(ok, StandardCharsets.UTF_8).startsWith("HTTP/1.1 200"), + "a tab is legal inside a field value"); + } + + @Test + @DisplayName("a response header whose name is not a token never reaches the wire") + void malformedResponseHeaderNamesAreDropped() throws Exception { + // /rawheader asks for four extra headers, three of which are not field + // names. A space inside a name makes a field line no peer can read; a + // LEADING space is obsolete line folding, which appends the text to the + // PREVIOUS header instead, so a handler's header can silently rewrite one + // the server owns; a colon just ends the name early and renames the field. + byte[] response = raw("GET /rawheader HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + String text = new String(response, StandardCharsets.UTF_8); + String head = text.substring(0, text.indexOf("\r\n\r\n") + 4); + assertTrue(head.startsWith("HTTP/1.1 200"), "the reply should be a 200:\n" + text); + assertTrue(head.indexOf("X-Good: ok") >= 0, + "a well formed extra header must still be sent:\n" + head); + assertEquals(-1, head.indexOf("space-in-name"), + "a name with a space in it is not a field name:\n" + head); + assertEquals(-1, head.indexOf("obsolete-folding"), + "a name with a leading space folds into the header before it:\n" + head); + assertEquals(-1, head.indexOf("colon-in-name"), + "a colon ends the name early and renames the field:\n" + head); + assertTrue(text.endsWith("raw"), "the body must still be intact:\n" + text); + } + + @Test + @DisplayName("a 205 carries neither content nor a length that claims any") + void resetContentIsBodilessAndZeroLength() throws Exception { + // RFC 9110 15.3.6: a Reset Content response cannot contain content and + // ends at the header section. Suppressing the body is only half of it -- + // advertising the SUPPRESSED body's length would leave a keep-alive + // client waiting for bytes that are never sent, which is the same + // desync from the other direction. + byte[] response = raw("GET /reset HTTP/1.1\r\nHost: x\r\n\r\n" + + "GET /healthz HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + String text = new String(response, StandardCharsets.UTF_8); + assertTrue(text.startsWith("HTTP/1.1 205"), "the first reply should be a 205:\n" + text); + assertEquals(-1, text.indexOf("junk"), "a 205 must not carry content:\n" + text); + String head = text.substring(0, text.indexOf("\r\n\r\n") + 4); + assertEquals(-1, head.indexOf("Content-Length: 4"), + "a 205 must not advertise the length it did not send:\n" + head); + assertEquals(2, countOccurrences(text, "HTTP/1.1 "), + "both replies must be readable back to back:\n" + text); + } + + @Test + @DisplayName("a Connection option is matched as a whole token, not a substring") + void connectionOptionsAreWholeTokens() throws Exception { + // "disclose" contains "close". Read as a substring it shut the connection, + // so an extension token this server has never heard of decided the framing. + // The second request is only answered if the first did not close. + byte[] response = raw("GET /healthz HTTP/1.1\r\nHost: x\r\nConnection: disclose\r\n\r\n" + + "GET /healthz HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + String text = new String(response, StandardCharsets.UTF_8); + assertEquals(2, countOccurrences(text, "HTTP/1.1 "), + "Connection: disclose is not Connection: close, so the connection had to " + + "stay open for the second request:\n" + text); + } + + @Test + @DisplayName("Content-Length together with Transfer-Encoding is refused") + void refusesConflictingFraming() throws Exception { + // Two framings in one request is how a request is smuggled past a proxy + // that believes one of them and a server that believes the other. + byte[] response = raw("POST /pet HTTP/1.1\r\nHost: x\r\nContent-Length: 6\r\n" + + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n0\r\n\r\n"); + assertEquals(400, statusOf(response)); + } + + @Test + @DisplayName("two different Content-Length values are refused") + void refusesDuplicateContentLength() throws Exception { + byte[] response = raw("POST /pet HTTP/1.1\r\nHost: x\r\nContent-Length: 5\r\n" + + "Content-Length: 6\r\nConnection: close\r\n\r\nhello"); + assertEquals(400, statusOf(response), + "disagreeing lengths must be refused, not guessed at"); + } + + @Test + @DisplayName("an HTTP/1.1 request without Host is refused") + void requiresHostHeader() throws Exception { + byte[] response = raw("GET /healthz HTTP/1.1\r\nConnection: close\r\n\r\n"); + assertEquals(400, statusOf(response)); + } + + @Test + @DisplayName("an obsolete folded header is refused") + void refusesObsoleteLineFolding() throws Exception { + // Folding is how two parsers are made to disagree about where a header + // ends; RFC 9112 says a server must reject it. + byte[] response = raw("GET /healthz HTTP/1.1\r\nHost: x\r\nX-Fold: a\r\n b\r\n" + + "Connection: close\r\n\r\n"); + assertEquals(400, statusOf(response)); + } + + @Test + @DisplayName("HTTP/1.0 closes unless the client asks to keep alive") + void httpTenClosesByDefault() throws Exception { + byte[] response = raw("GET /healthz HTTP/1.0\r\n\r\n"); + assertEquals(200, statusOf(response)); + String connection = header(response, "Connection"); + assertTrue(connection == null || "close".equalsIgnoreCase(connection), + "HTTP/1.0 defaults to close, got Connection: " + connection); + } + + @Test + @DisplayName("Expect: 100-continue gets an interim response") + void honoursExpectContinue() throws Exception { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(3000); + try { + OutputStream out = socket.getOutputStream(); + String payload = "{\"name\":\"Expectant\"}"; + out.write(("POST /pet HTTP/1.1\r\nHost: x\r\nContent-Length: " + payload.length() + + "\r\nExpect: 100-continue\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.UTF_8)); + out.flush(); + // The client is entitled to wait here. A server that never answers + // makes every such client pay its whole timeout before sending. + byte[] interim = new byte[64]; + int n = socket.getInputStream().read(interim); + String head = new String(interim, 0, Math.max(n, 0), StandardCharsets.UTF_8); + assertTrue(head.startsWith("HTTP/1.1 100"), + "expected an interim 100 Continue, got: " + head.trim()); + } finally { + socket.close(); + } + } + + @Test + @DisplayName("an unknown method is refused rather than routed") + void refusesUnknownMethod() throws Exception { + // Methods are case-sensitive, so "get" is not GET. + byte[] response = raw("get /healthz HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"); + int code = statusOf(response); + assertTrue(code == 400 || code == 501, + "a method that is not a known verb must be refused, got " + code); + } + + @Test + @DisplayName("a silent client is shed and does not hold a worker") + void shedsIdleConnections() throws Exception { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(15000); + try { + // Enough to be handed to a worker, never enough to be a request. + socket.getOutputStream().write("GET /healthz HTTP/1.1\r\nHost: x\r\n" + .getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + long started = System.currentTimeMillis(); + int first = socket.getInputStream().read(); + long elapsed = System.currentTimeMillis() - started; + assertEquals(-1, first, "the connection should be closed, not answered"); + assertTrue(elapsed < 12000, "the deadline should have shed it, took " + elapsed + "ms"); + } finally { + socket.close(); + } + // And the server is still healthy afterwards. + assertEquals(200, status(request("GET", "/healthz", null, null))); + } + + // ------------------------------------------------------------------ + // Resilience + // + // Every other test here is a prompt client: it sends a whole request and + // reads the whole reply at once. Two shipped regressions lived precisely in + // what that never exercises -- a client that writes slowly pinned the thread + // serving it, and a client that reads slowly had its response truncated, + // because the non-blocking descriptors introduced for virtual threads made + // both paths meet EAGAIN for the first time. These two hold that ground. + // ------------------------------------------------------------------ + + @Test + @DisplayName("a response larger than the socket buffer survives a slow reader") + void slowReaderReceivesTheWholeResponse() throws Exception { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(20000); + try { + socket.getOutputStream().write(("GET /static/huge.bin HTTP/1.1\r\n" + + "Host: 127.0.0.1\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + InputStream in = socket.getInputStream(); + + // Read just the head, then stop reading. The server keeps writing + // until the kernel's send buffer is full and its next write answers + // EAGAIN -- the state the whole test exists to produce. + ByteArrayOutputStream head = new ByteArrayOutputStream(); + String headText; + for (;;) { + int c = in.read(); + assertTrue(c >= 0, "the connection closed before the headers ended"); + head.write(c); + headText = new String(head.toByteArray(), StandardCharsets.UTF_8); + if (headText.endsWith("\r\n\r\n")) { + break; + } + } + assertTrue(headText.startsWith("HTTP/1.1 200"), "unexpected head: " + headText); + Thread.sleep(750); + + // Now drain, and count. A truncation shows up as a short total, and + // a corrupted one as a byte that is not where it should be. + byte[] chunk = new byte[16 * 1024]; + long total = 0; + for (;;) { + int n = in.read(chunk); + if (n < 0) { + break; + } + for (int i = 0; i < n; i++) { + long at = total + i; + assertEquals((byte) ((at * 31) & 0xff), chunk[i], + "the body is corrupt at offset " + at); + } + total += n; + } + assertEquals(HUGE_BYTES, total, + "the response was truncated: got " + total + " of " + HUGE_BYTES + + " bytes, which is what treating EAGAIN as a failure does"); + } finally { + socket.close(); + } + } + + @Test + @DisplayName("a head dribbled a byte at a time is cut off rather than held forever") + void aDribbledRequestHeadIsCutOff() throws Exception { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(30000); + try { + OutputStream out = socket.getOutputStream(); + out.write("GET /healthz HTTP/1.1\r\nHost: x\r\n".getBytes(StandardCharsets.UTF_8)); + out.flush(); + // A byte inside every socket-timeout window. SO_RCVTIMEO restarts on + // each one, so this alone would keep its worker for as long as the + // client cared to continue; only a deadline measured from the head's + // FIRST byte ends it. + long started = System.currentTimeMillis(); + String filler = "X-Pad: "; + boolean closed = false; + for (int i = 0; i < 40 && !closed; i++) { + try { + out.write(filler.charAt(i % filler.length())); + out.flush(); + } catch (IOException dropped) { + closed = true; + break; + } + Thread.sleep(500); + if (socket.getInputStream().available() > 0) { + closed = true; + } + } + long elapsed = System.currentTimeMillis() - started; + assertTrue(closed, "the server accepted a head dribbled for " + elapsed + + "ms without ever ending it"); + // CN1_HTTP_TIMEOUT_MS is 4000 for this fixture, so the deadline should + // land well inside this. Generous, because a loaded runner is slow. + assertTrue(elapsed < 20000, "the head was cut off, but only after " + + elapsed + "ms"); + } finally { + socket.close(); + } + assertEquals(200, status(request("GET", "/healthz", null, null))); + } + + @Test + @DisplayName("clients that never finish a request do not starve the ones that do") + void partialRequestsDoNotStarveOtherClients() throws Exception { + // Comfortably more than the worker pool, so if a half-written request + // holds the thread that serves it, nothing is left to answer the probe. + final int stalled = 64; + Socket[] sockets = new Socket[stalled]; + try { + for (int i = 0; i < stalled; i++) { + sockets[i] = new Socket(); + sockets[i].connect(new InetSocketAddress("127.0.0.1", port), 5000); + // A request head that has begun and will never end. + sockets[i].getOutputStream().write( + ("GET /healthz HTTP/1.1\r\nHost: 127.0.0.1\r\nX-Stall: " + i + "\r\n") + .getBytes(StandardCharsets.UTF_8)); + sockets[i].getOutputStream().flush(); + } + // Promptly, before the idle deadline sheds any of them. + long started = System.currentTimeMillis(); + assertEquals(200, status(request("GET", "/healthz", null, null)), + "a healthy request must still be answered while " + stalled + + " connections sit mid-request"); + long elapsed = System.currentTimeMillis() - started; + assertTrue(elapsed < 5000, + "the probe waited " + elapsed + "ms, so the stalled connections " + + "are holding the threads that should have served it"); + } finally { + for (int i = 0; i < stalled; i++) { + if (sockets[i] != null) { + try { + sockets[i].close(); + } catch (IOException ignored) { + // the server may already have shed it + } + } + } + } + // The shed connections must not have left the server damaged. + assertEquals(200, status(request("GET", "/healthz", null, null))); + } + + // ------------------------------------------------------------------ + // TLS + // + // The handshake, the record layer and the user-space copy that replaces + // sendfile on a TLS connection. ALPN negotiation is NOT covered: this module + // targets 1.8, where the client-side API to request a protocol and read back + // what was chosen does not exist, so a test for it could only ever skip. + // ------------------------------------------------------------------ + + @Test + @DisplayName("a request is served over TLS") + void tlsServesARequest() throws Exception { + SSLSocket socket = openTls(); + try { + socket.startHandshake(); + socket.getOutputStream().write(("GET /healthz HTTP/1.1\r\nHost: localhost\r\n" + + "Connection: close\r\n\r\n").getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + String response = readFully(socket.getInputStream()); + assertTrue(response.startsWith("HTTP/1.1 200"), + "TLS did not serve the request: " + response); + } finally { + socket.close(); + } + } + + @Test + @DisplayName("a kept-alive TLS connection left idle is shed, not held for ever") + void tlsIdleKeepAliveConnectionsAreShed() throws Exception { + // The case that has no thread in recv: ONE request is answered, the + // connection goes back to the reactor, and the client then says nothing. + // A connection that never speaks at all is held by a worker inside recv + // and shed by SO_RCVTIMEO, so it proves nothing about the reactor -- an + // earlier version of this test did exactly that and passed with the sweep + // removed. The TLS server runs on the pool, which had no idle deadline of + // its own, so these accumulated to MAX_CONNECTIONS and every later client + // was refused. + SSLSocket socket = openTls(); + try { + socket.startHandshake(); + socket.setSoTimeout(30000); + socket.getOutputStream().write(("GET /healthz HTTP/1.1\r\nHost: localhost\r\n" + + "Connection: keep-alive\r\n\r\n").getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + + InputStream in = socket.getInputStream(); + ByteArrayOutputStream head = new ByteArrayOutputStream(); + String text; + for (;;) { + int c = in.read(); + assertTrue(c >= 0, "the first reply never arrived"); + head.write(c); + text = new String(head.toByteArray(), StandardCharsets.UTF_8); + if (text.endsWith("\r\n\r\n")) { + break; + } + } + assertTrue(text.startsWith("HTTP/1.1 200"), "unexpected reply: " + text); + + // Answered and parked. Now nothing is reading it on the server side. + // Short client-side reads so the wait can END with this test's own + // sentence: blocking for the whole window instead threw a bare + // SocketTimeoutException from the client, which says nothing about + // what the server did. + socket.setSoTimeout(2000); + long started = System.currentTimeMillis(); + boolean closed = false; + while (System.currentTimeMillis() - started < 25000) { + try { + if (in.read() < 0) { + closed = true; + break; + } + } catch (java.net.SocketTimeoutException stillOpen) { + // The server has not closed it yet; keep waiting. + } + } + long elapsed = System.currentTimeMillis() - started; + assertTrue(closed, "the parked keep-alive connection was still open after " + + elapsed + "ms, so nothing sheds a pooled connection once the " + + "reactor has it back"); + assertTrue(elapsed < 25000, + "the idle deadline should have shed it, took " + elapsed + "ms"); + assertTrue(elapsed > 500, "closed implausibly fast (" + elapsed + + "ms): the connection may not have been parked at all"); + } finally { + socket.close(); + } + assertEquals(200, status(request("GET", "/healthz", null, null))); + } + + @Test + @DisplayName("a large file survives a slow reader over TLS too") + void tlsSlowReaderReceivesTheWholeResponse() throws Exception { + // TLS has no sendfile path -- the bytes have to be encrypted in user + // space -- so this covers the read/write copy that sendfile bypasses. + SSLSocket socket = openTls(); + try { + socket.startHandshake(); + socket.getOutputStream().write(("GET /static/huge.bin HTTP/1.1\r\n" + + "Host: localhost\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + InputStream in = socket.getInputStream(); + ByteArrayOutputStream head = new ByteArrayOutputStream(); + String headText; + for (;;) { + int c = in.read(); + assertTrue(c >= 0, "the connection closed before the headers ended"); + head.write(c); + headText = new String(head.toByteArray(), StandardCharsets.UTF_8); + if (headText.endsWith("\r\n\r\n")) { + break; + } + } + assertTrue(headText.startsWith("HTTP/1.1 200"), "unexpected head: " + headText); + Thread.sleep(750); + byte[] chunk = new byte[16 * 1024]; + long total = 0; + for (;;) { + int n = in.read(chunk); + if (n < 0) { + break; + } + total += n; + } + assertEquals(HUGE_BYTES, total, "the TLS response was truncated"); + } finally { + socket.close(); + } + } + + /** + * Connects to the TLS port, trusting the throwaway self-signed certificate. + * + * Skips rather than fails when no TLS server came up: a machine without + * openssl cannot make a certificate, and that says nothing about the server. + */ + private SSLSocket openTls() throws Exception { + Assumptions.assumeTrue(tlsServer != null && tlsPort != 0, + "no TLS server (openssl unavailable, or it did not start)"); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, new TrustManager[]{ new X509TrustManager() { + public void checkClientTrusted(X509Certificate[] chain, String authType) { } + public void checkServerTrusted(X509Certificate[] chain, String authType) { } + public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } }, null); + SSLSocket socket = (SSLSocket) context.getSocketFactory() + .createSocket("127.0.0.1", tlsPort); + socket.setSoTimeout(20000); + return socket; + } + + // ------------------------------------------------------------------ + // HTTP/2 + // ------------------------------------------------------------------ + + @Test + @DisplayName("cleartext HTTP/2 by prior knowledge serves a request") + void http2CleartextRequest() throws Exception { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(10000); + try { + OutputStream out = socket.getOutputStream(); + out.write("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + out.write(frame(4, 0, 0, new byte[0])); // empty SETTINGS + ByteArrayOutputStream block = new ByteArrayOutputStream(); + hpackLiteral(block, ":method", "GET"); + hpackLiteral(block, ":path", "/healthz"); + hpackLiteral(block, ":scheme", "http"); + hpackLiteral(block, ":authority", "127.0.0.1"); + // END_STREAM | END_HEADERS: a GET with no body is complete at once. + out.write(frame(1, 0x05, 1, block.toByteArray())); + out.flush(); + + boolean sawHeaders = false; + boolean sawData = false; + String data = ""; + long deadline = System.currentTimeMillis() + 8000; + InputStream in = socket.getInputStream(); + while (System.currentTimeMillis() < deadline && !(sawHeaders && sawData)) { + byte[] header = readExactly(in, 9); + if (header == null) { + break; + } + int length = ((header[0] & 0xff) << 16) | ((header[1] & 0xff) << 8) | (header[2] & 0xff); + int type = header[3] & 0xff; + byte[] payload = length == 0 ? new byte[0] : readExactly(in, length); + if (payload == null) { + break; + } + if (type == 1) { + sawHeaders = true; + // The status is HPACK-encoded; 200 is static-table index 8, + // which nghttp2 emits as the single byte 0x88. + assertTrue(payload.length > 0, "an empty HEADERS payload is not a response"); + assertEquals((byte) 0x88, payload[0], + "expected an indexed :status 200 as the first header"); + // RFC 9110 6.6.1 wants Date on every response, and the HTTP/1 + // writer sends it. This asserts the h2 path does too: a first + // attempt added the name and the value as separate entries, + // which Http2.headerLines() turned into two colonless lines the + // native parser dropped, and nothing here noticed. + assertTrue(hpackNameIndices(payload).contains(Integer.valueOf(33)), + "the HEADERS block carries no date (static name index 33)"); + } else if (type == 0) { + sawData = true; + data = new String(payload, StandardCharsets.UTF_8); + } else if (type == 7) { + fail("the server sent GOAWAY: " + new String(payload, StandardCharsets.UTF_8)); + } + } + assertTrue(sawHeaders, "no HEADERS frame came back"); + assertTrue(sawData, "no DATA frame came back"); + assertTrue(data.contains("\"status\":\"ok\""), data); + } finally { + socket.close(); + } + } + + @Test + @DisplayName("an h2 body over the ceiling is refused, and the ceiling is given back") + void http2BodiesAreBoundedAndReleased() throws Exception { + // The ceiling is reserved natively, in the same step as the allocation -- + // a limit tested in Java and enforced in C is two steps with a gap, and + // two sessions being processed at once both read the total below the + // ceiling and then both allocate. + // + // What a client can see is the two ends of that: a body over the ceiling + // is refused rather than served, and the reservation comes back when the + // body is done, so the NEXT request over the ceiling is refused for the + // same reason rather than because the first one is still charged. Without + // the release, request two would be refused at any size at all. + Assumptions.assumeTrue(smallUploadPort > 0, + "the small-ceiling server did not start"); + assertEquals(503, h2StatusFor(smallUploadPort, "/bulk?size=" + (6 * 1024 * 1024)), + "a body over the ceiling must be refused"); + assertEquals(200, h2StatusFor(smallUploadPort, "/bulk?size=1024"), + "a small body after it must still be served: the refusal must not " + + "have left its bytes charged"); + // THREE two-megabyte bodies against a four-megabyte ceiling. Each one is + // under it, but their sum is not, so they only all succeed if each + // reservation is released when its body finishes. A first version of this + // test asked for one 1KB and one 2MB body -- never reaching the ceiling + // cumulatively -- and so passed with every release deleted. + for (int i = 0; i < 3; i++) { + assertEquals(200, h2StatusFor(smallUploadPort, "/bulk?size=" + (2 * 1024 * 1024)), + "body " + i + " of three under the ceiling was refused, so an " + + "earlier one's reservation was never released"); + } + assertEquals(503, h2StatusFor(smallUploadPort, "/bulk?size=" + (6 * 1024 * 1024)), + "and the ceiling still applies afterwards"); + } + + /** The :status of one h2c GET, decoded from the HEADERS block. */ + private int h2StatusFor(int onPort, String path) throws Exception { + return h2StatusFor(onPort, path, "GET"); + } + + private int h2StatusFor(int onPort, String path, String method) throws Exception { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", onPort), 5000); + socket.setSoTimeout(20000); + try { + OutputStream out = socket.getOutputStream(); + out.write("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + out.write(frame(4, 0, 0, new byte[0])); + byte[] windowUpdate = new byte[4]; + int increment = 8 * 1024 * 1024; + windowUpdate[0] = (byte) ((increment >> 24) & 0x7f); + windowUpdate[1] = (byte) ((increment >> 16) & 0xff); + windowUpdate[2] = (byte) ((increment >> 8) & 0xff); + windowUpdate[3] = (byte) (increment & 0xff); + out.write(frame(8, 0, 0, windowUpdate)); + ByteArrayOutputStream block = new ByteArrayOutputStream(); + hpackLiteral(block, ":method", method); + hpackLiteral(block, ":path", path); + hpackLiteral(block, ":scheme", "http"); + hpackLiteral(block, ":authority", "127.0.0.1"); + out.write(frame(1, 0x05, 1, block.toByteArray())); + out.flush(); + out.write(frame(8, 0, 1, windowUpdate)); + out.flush(); + + long deadline = System.currentTimeMillis() + 20000; + InputStream in = socket.getInputStream(); + boolean done = false; + int status = -1; + while (System.currentTimeMillis() < deadline && !done) { + byte[] header = readExactly(in, 9); + if (header == null) { + break; + } + int length = ((header[0] & 0xff) << 16) | ((header[1] & 0xff) << 8) + | (header[2] & 0xff); + int type = header[3] & 0xff; + int flags = header[4] & 0xff; + byte[] payload = length == 0 ? new byte[0] : readExactly(in, length); + if (payload == null) { + break; + } + if (type == 1 && payload.length > 0) { + status = hpackStatus(payload); + done = (flags & 0x01) != 0; + } else if (type == 0) { + done = (flags & 0x01) != 0; + } else if (type == 7) { + fail("the server sent GOAWAY: " + new String(payload, StandardCharsets.UTF_8)); + } + } + return status; + } finally { + socket.close(); + } + } + + @Test + @DisplayName("a large h2 body survives the bounded output buffer") + void http2DeliversABodyLargerThanTheOutputBuffer() throws Exception { + // The serialisation buffer is capped, and the send callback answers + // WOULDBLOCK once it is full so nghttp2 stops and keeps the rest. That + // only works because drain() pumps again after emptying; a cap without + // the re-pump would truncate every response bigger than the buffer, and + // the small bodies every other test sends would never notice. + int size = 3 * 1024 * 1024; + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(20000); + try { + OutputStream out = socket.getOutputStream(); + out.write("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + out.write(frame(4, 0, 0, new byte[0])); + // Raise the connection window so the whole body is writable at once: + // that is the condition under which nghttp2 fills the buffer in a + // single pump, which is exactly what the cap has to survive. + byte[] windowUpdate = new byte[4]; + int increment = size + 65536; + windowUpdate[0] = (byte) ((increment >> 24) & 0x7f); + windowUpdate[1] = (byte) ((increment >> 16) & 0xff); + windowUpdate[2] = (byte) ((increment >> 8) & 0xff); + windowUpdate[3] = (byte) (increment & 0xff); + out.write(frame(8, 0, 0, windowUpdate)); + ByteArrayOutputStream block = new ByteArrayOutputStream(); + hpackLiteral(block, ":method", "GET"); + hpackLiteral(block, ":path", "/bulk?size=" + size); + hpackLiteral(block, ":scheme", "http"); + hpackLiteral(block, ":authority", "127.0.0.1"); + out.write(frame(1, 0x05, 1, block.toByteArray())); + out.flush(); + out.write(frame(8, 0, 1, windowUpdate)); // and the stream window + out.flush(); + + ByteArrayOutputStream received = new ByteArrayOutputStream(); + boolean endStream = false; + long deadline = System.currentTimeMillis() + 20000; + InputStream in = socket.getInputStream(); + while (System.currentTimeMillis() < deadline && !endStream) { + byte[] header = readExactly(in, 9); + if (header == null) { + break; + } + int length = ((header[0] & 0xff) << 16) | ((header[1] & 0xff) << 8) | (header[2] & 0xff); + int type = header[3] & 0xff; + int flags = header[4] & 0xff; + byte[] payload = length == 0 ? new byte[0] : readExactly(in, length); + if (payload == null) { + break; + } + if (type == 0) { + received.write(payload); + endStream = (flags & 0x01) != 0; + } else if (type == 7) { + fail("the server sent GOAWAY: " + new String(payload, StandardCharsets.UTF_8)); + } + } + assertTrue(endStream, "the stream never ended; got " + received.size() + " of " + size); + assertEquals(size, received.size(), "the body was truncated"); + byte[] bytes = received.toByteArray(); + for (int iter = 0; iter < bytes.length; iter++) { + if (bytes[iter] != (byte) ('a' + (iter % 26))) { + fail("byte " + iter + " is wrong: the frames were reassembled out of order"); + } + } + } finally { + socket.close(); + } + } + + @Test + @DisplayName("an h2 HEAD of a static file closes its descriptor exactly once") + void http2HeadOfAFileClosesItOnce() throws Exception { + // A review reported this as a LEAK: the bodiless branch never closes the + // descriptor, so repeated HEADs exhaust the process. It is not -- the + // responseBodyFor() call below that branch closes it in a finally, which + // is why it is called at all on a path that wants no body. + // + // Adding a close there anyway made it a DOUBLE close, and this is what + // says so: the count runs NEGATIVE, one per request. That is worse than + // the reported bug, because a descriptor number is reusable the moment + // the first close returns and the second lands on whoever took it. + // + // Both directions are pinned here on purpose. Zero is the answer; a + // positive number is the leak the review predicted and a negative one is + // the "fix" for it. + int before = openStaticFiles(); + for (int i = 0; i < 10; i++) { + assertEquals(200, h2StatusFor(port, "/static/big.bin", "HEAD"), + "the HEAD itself must be answered"); + } + assertEquals(before, openStaticFiles(), + "ten HEADs must leave the descriptor count exactly where it was"); + } + + /** The server's own count of descriptors handed out and not yet closed. */ + private int openStaticFiles() throws Exception { + String metrics = body(request("GET", "/healthz", null, null)); + int at = metrics.indexOf("\"openStaticFiles\""); + assertTrue(at >= 0, "the server does not report openStaticFiles: " + metrics); + int colon = metrics.indexOf(':', at); + int end = colon + 1; + // The MINUS matters. A first version scanned for digits only, so the -10 + // that a double close produces was read as 10 and reported as the leak + // being looked for -- the measurement agreed with the hypothesis by + // discarding the character that disproved it. + while (end < metrics.length() && "-0123456789".indexOf(metrics.charAt(end)) < 0) { + end++; + } + int start = end; + if (end < metrics.length() && metrics.charAt(end) == '-') { + end++; + } + while (end < metrics.length() && "0123456789".indexOf(metrics.charAt(end)) >= 0) { + end++; + } + return Integer.parseInt(metrics.substring(start, end)); + } + + @Test + @DisplayName("a HEAD over h2 reports the length a GET would send") + void http2HeadReportsRealLength() throws Exception { + // The HTTP/1 writer keeps the representation length for a HEAD, because + // describing what is NOT being sent is the whole point of asking. The + // HTTP/2 path did not, so one static file answered a size over one + // protocol and nothing over the other, from the same handler. + // + // The presence of the header is what is asserted here: its value is + // HPACK-encoded and may be Huffman-coded, and headReportsRealLength + // already pins the exact number over HTTP/1. Static index 28 is + // content-length (RFC 7541 Appendix A). + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(10000); + try { + OutputStream out = socket.getOutputStream(); + out.write("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + out.write(frame(4, 0, 0, new byte[0])); + ByteArrayOutputStream block = new ByteArrayOutputStream(); + hpackLiteral(block, ":method", "HEAD"); + hpackLiteral(block, ":path", "/static/big.bin"); + hpackLiteral(block, ":scheme", "http"); + hpackLiteral(block, ":authority", "127.0.0.1"); + out.write(frame(1, 0x05, 1, block.toByteArray())); + out.flush(); + + byte[] responseHeaders = null; + long deadline = System.currentTimeMillis() + 8000; + InputStream in = socket.getInputStream(); + while (System.currentTimeMillis() < deadline && responseHeaders == null) { + byte[] header = readExactly(in, 9); + if (header == null) { + break; + } + int length = ((header[0] & 0xff) << 16) | ((header[1] & 0xff) << 8) + | (header[2] & 0xff); + int type = header[3] & 0xff; + byte[] payload = length == 0 ? new byte[0] : readExactly(in, length); + if (payload == null) { + break; + } + if (type == 1) { + responseHeaders = payload; + } + } + assertNotNull(responseHeaders, "no HEADERS frame came back for the HEAD"); + assertTrue(hpackNameIndices(responseHeaders).contains(Integer.valueOf(28)), + "a HEAD over h2 must report the length it is not sending"); + } finally { + socket.close(); + } + } + + @Test + @DisplayName("a HEAD over h2 reports the length of a DEFERRED json body") + void http2HeadReportsDeferredJsonLength() throws Exception { + // respondJson leaves the value unserialised so the HTTP/1 writer can render + // it straight into the connection buffer, which means response.body is + // EMPTY. The h2 HEAD calculation measured that array and answered + // content-length: 0 for a representation that is not -- and the earlier h2 + // HEAD fix did not close it, because it only learned about file and eager + // byte-array bodies. /deferred is the only route shaped this way. + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(10000); + try { + OutputStream out = socket.getOutputStream(); + out.write("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + out.write(frame(4, 0, 0, new byte[0])); + ByteArrayOutputStream block = new ByteArrayOutputStream(); + hpackLiteral(block, ":method", "HEAD"); + hpackLiteral(block, ":path", "/deferred"); + hpackLiteral(block, ":scheme", "http"); + hpackLiteral(block, ":authority", "127.0.0.1"); + out.write(frame(1, 0x05, 1, block.toByteArray())); + out.flush(); + + byte[] responseHeaders = null; + long deadline = System.currentTimeMillis() + 8000; + InputStream in = socket.getInputStream(); + while (System.currentTimeMillis() < deadline && responseHeaders == null) { + byte[] header = readExactly(in, 9); + if (header == null) { + break; + } + int length = ((header[0] & 0xff) << 16) | ((header[1] & 0xff) << 8) + | (header[2] & 0xff); + int type = header[3] & 0xff; + byte[] payload = length == 0 ? new byte[0] : readExactly(in, length); + if (payload == null) { + break; + } + if (type == 1) { + responseHeaders = payload; + } + } + assertNotNull(responseHeaders, "no HEADERS frame came back for the HEAD"); + long described = hpackNumericValue(responseHeaders, 28); + assertTrue(described > 0, + "a HEAD of a deferred json body reported " + described + + " instead of the length a GET would send"); + // And it is the length a GET really sends, not merely nonzero. + String json = body(request("GET", "/deferred", null, null)); + assertEquals(json.getBytes(StandardCharsets.UTF_8).length, described, + "the described length is not the one a GET returns"); + } finally { + socket.close(); + } + } + + @Test + @DisplayName("a HEAD of a 204 over h2 carries no length either") + void http2HeadOfABodilessStatusHasNoLength() throws Exception { + // The HEAD rule and the STATUS rule meet here. A HEAD describes the + // representation it is not sending, but a 204 has none to describe and + // RFC 9110 6.4.1 forbids the field outright -- which the HTTP/1 writer + // already honours. Adding it unconditionally on the h2 path made one + // response valid over one protocol and invalid over the other, the exact + // divergence the HEAD fix existed to remove. + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(10000); + try { + OutputStream out = socket.getOutputStream(); + out.write("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + out.write(frame(4, 0, 0, new byte[0])); + ByteArrayOutputStream block = new ByteArrayOutputStream(); + hpackLiteral(block, ":method", "HEAD"); + hpackLiteral(block, ":path", "/nocontent"); + hpackLiteral(block, ":scheme", "http"); + hpackLiteral(block, ":authority", "127.0.0.1"); + out.write(frame(1, 0x05, 1, block.toByteArray())); + out.flush(); + + byte[] responseHeaders = null; + long deadline = System.currentTimeMillis() + 8000; + InputStream in = socket.getInputStream(); + while (System.currentTimeMillis() < deadline && responseHeaders == null) { + byte[] header = readExactly(in, 9); + if (header == null) { + break; + } + int length = ((header[0] & 0xff) << 16) | ((header[1] & 0xff) << 8) + | (header[2] & 0xff); + int type = header[3] & 0xff; + byte[] payload = length == 0 ? new byte[0] : readExactly(in, length); + if (payload == null) { + break; + } + if (type == 1) { + responseHeaders = payload; + } + } + assertNotNull(responseHeaders, "no HEADERS frame came back for the HEAD"); + assertTrue(!hpackNameIndices(responseHeaders).contains(Integer.valueOf(28)), + "a 204 must not carry content-length, over either protocol"); + } finally { + socket.close(); + } + } + + @Test + @DisplayName("an HTTP/1.1 request still works on the same port as h2c") + void httpOneStillWorksAlongsideHttp2() throws Exception { + // The preface detector must not swallow ordinary requests: "GET" diverges + // from "PRI" at the second byte and has to fall straight through. + assertEquals(200, status(request("GET", "/healthz", null, null))); + } + + /** An HTTP/2 frame: 3-byte length, type, flags, 4-byte stream id, payload. */ + /** + * The HPACK static name indices used by a HEADERS block. + * + * Walks the field representations rather than searching for a byte, because a + * Huffman-encoded value can contain any byte it likes. Only the shapes nghttp2 + * emits for a response are handled; anything else ends the walk. + */ + private static java.util.Set hpackNameIndices(byte[] block) { + java.util.Set names = new java.util.HashSet(); + int at = 0; + while (at < block.length) { + int b = block[at] & 0xff; + int prefixBits; + boolean hasValue; + if ((b & 0x80) != 0) { + prefixBits = 7; // indexed field: name AND value + hasValue = false; + } else if ((b & 0xC0) == 0x40) { + prefixBits = 6; // literal, incremental indexing + hasValue = true; + } else if ((b & 0xE0) == 0x20) { + prefixBits = 5; // dynamic table size update + hasValue = false; + } else { + prefixBits = 4; // literal, without / never indexed + hasValue = true; + } + int[] cursor = { at }; + int index = hpackInteger(block, cursor, prefixBits); + if (index < 0) { + break; + } + names.add(Integer.valueOf(index)); + at = cursor[0]; + if (index == 0) { + at = hpackSkipString(block, at); // the name is spelled out + if (at < 0) { + break; + } + } + if (hasValue) { + at = hpackSkipString(block, at); + if (at < 0) { + break; + } + } + } + return names; + } + + /** + * The value of one static-index field, read as a number, or -1 if absent. + * + * Only content-length is asked for here and its value is always digits, so the + * Huffman side needs the ten digit codes and nothing more (RFC 7541 Appendix B: + * 0, 1 and 2 are five bits, 3 through 9 are six). nghttp2 picks Huffman only + * when it is strictly shorter, which for digits starts at three of them -- so a + * test that read the raw bytes alone would pass on short lengths and quietly + * stop asserting on longer ones. + */ + private static long hpackNumericValue(byte[] block, int nameIndex) { + int at = 0; + while (at < block.length) { + int b = block[at] & 0xff; + int prefixBits; + boolean hasValue; + if ((b & 0x80) != 0) { + prefixBits = 7; + hasValue = false; + } else if ((b & 0xC0) == 0x40) { + prefixBits = 6; + hasValue = true; + } else if ((b & 0xE0) == 0x20) { + prefixBits = 5; + hasValue = false; + } else { + prefixBits = 4; + hasValue = true; + } + int[] cursor = { at }; + int index = hpackInteger(block, cursor, prefixBits); + if (index < 0) { + return -1; + } + at = cursor[0]; + if (index == 0) { + at = hpackSkipString(block, at); + if (at < 0) { + return -1; + } + } + if (hasValue) { + int valueAt = at; + at = hpackSkipString(block, at); + if (at < 0) { + return -1; + } + if (index == nameIndex) { + return hpackDigits(block, valueAt); + } + } + } + return -1; + } + + /** + * The :status of a HEADERS block, decoded rather than guessed. + * + * A first version read "first byte is 0x88, so 200, otherwise 503". That is + * true only when nghttp2 happens to emit the indexed form first, and a 200 + * carrying content-length did not -- so a perfectly good response was + * reported as the failure the test was looking for, which is the worst + * direction for a guess to be wrong in. + * + * :status occupies static-table entries 8 through 14 (200, 204, 206, 304, + * 400, 404, 500); anything else arrives as a literal against name index 8. + */ + private static int hpackStatus(byte[] block) { + int[] indexed = { 0, 0, 0, 0, 0, 0, 0, 0, 200, 204, 206, 304, 400, 404, 500 }; + int at = 0; + while (at < block.length) { + int b = block[at] & 0xff; + int prefixBits; + boolean hasValue; + if ((b & 0x80) != 0) { + prefixBits = 7; + hasValue = false; + } else if ((b & 0xC0) == 0x40) { + prefixBits = 6; + hasValue = true; + } else if ((b & 0xE0) == 0x20) { + prefixBits = 5; + hasValue = false; + } else { + prefixBits = 4; + hasValue = true; + } + int[] cursor = { at }; + int index = hpackInteger(block, cursor, prefixBits); + if (index < 0) { + return -1; + } + at = cursor[0]; + if (!hasValue && index >= 8 && index <= 14) { + return indexed[index]; + } + if (index == 0) { + at = hpackSkipString(block, at); + if (at < 0) { + return -1; + } + } + if (hasValue) { + int valueAt = at; + at = hpackSkipString(block, at); + if (at < 0) { + return -1; + } + if (index == 8) { + return (int) hpackDigits(block, valueAt); + } + } + } + return -1; + } + + /** A length-prefixed string of digits, raw or Huffman, as a number. */ + private static long hpackDigits(byte[] block, int at) { + boolean huffman = (block[at] & 0x80) != 0; + int[] cursor = { at }; + int length = hpackInteger(block, cursor, 7); + if (length < 0 || cursor[0] + length > block.length) { + return -1; + } + StringBuilder text = new StringBuilder(); + if (!huffman) { + for (int iter = 0; iter < length; iter++) { + text.append((char) (block[cursor[0] + iter] & 0xff)); + } + } else { + int bits = length * 8; + int position = 0; + while (bits - position >= 5) { + int five = hpackBits(block, cursor[0], position, 5); + if (five <= 2) { // 00000, 00001, 00010 + text.append((char) ('0' + five)); + position += 5; + continue; + } + if (bits - position < 6) { + break; // what is left is padding + } + int six = hpackBits(block, cursor[0], position, 6); + if (six < 0x19 || six > 0x1f) { // 011001 .. 011111 + return -1; // not a digit: give up loudly + } + text.append((char) ('3' + (six - 0x19))); + position += 6; + } + } + try { + return Long.parseLong(text.toString()); + } catch (NumberFormatException notANumber) { + return -1; + } + } + + /** `count` bits starting `position` bits into the bytes at `from`. */ + private static int hpackBits(byte[] block, int from, int position, int count) { + int value = 0; + for (int iter = 0; iter < count; iter++) { + int bit = position + iter; + int b = block[from + (bit >> 3)] & 0xff; + value = (value << 1) | ((b >> (7 - (bit & 7))) & 1); + } + return value; + } + + /** RFC 7541 5.1, with the cursor left just past the integer. */ + private static int hpackInteger(byte[] block, int[] cursor, int prefixBits) { + int at = cursor[0]; + if (at >= block.length) { + return -1; + } + int mask = (1 << prefixBits) - 1; + int value = block[at++] & mask; + if (value == mask) { + int shift = 0; + for (;;) { + if (at >= block.length) { + return -1; + } + int next = block[at++] & 0xff; + value += (next & 0x7f) << shift; + shift += 7; + if ((next & 0x80) == 0) { + break; + } + } + } + cursor[0] = at; + return value; + } + + /** Skips a length-prefixed (possibly Huffman) string, or -1 if it runs out. */ + private static int hpackSkipString(byte[] block, int at) { + int[] cursor = { at }; + int length = hpackInteger(block, cursor, 7); + if (length < 0 || cursor[0] + length > block.length) { + return -1; + } + return cursor[0] + length; + } + + private static byte[] frame(int type, int flags, int streamId, byte[] payload) { + byte[] out = new byte[9 + payload.length]; + out[0] = (byte) ((payload.length >>> 16) & 0xff); + out[1] = (byte) ((payload.length >>> 8) & 0xff); + out[2] = (byte) (payload.length & 0xff); + out[3] = (byte) type; + out[4] = (byte) flags; + out[5] = (byte) ((streamId >>> 24) & 0x7f); + out[6] = (byte) ((streamId >>> 16) & 0xff); + out[7] = (byte) ((streamId >>> 8) & 0xff); + out[8] = (byte) (streamId & 0xff); + System.arraycopy(payload, 0, out, 9, payload.length); + return out; + } + + /** + * One HPACK "literal header field without indexing, new name", uncompressed. + * Writing a full HPACK encoder into a test would be testing the test; this is + * the one form every decoder must accept. + */ + private static void hpackLiteral(ByteArrayOutputStream out, String name, String value) { + byte[] n = name.getBytes(StandardCharsets.UTF_8); + byte[] v = value.getBytes(StandardCharsets.UTF_8); + out.write(0x00); + out.write(n.length); // H=0, length < 127 for every name used here + out.write(n, 0, n.length); + out.write(v.length); + out.write(v, 0, v.length); + } + + private static byte[] readExactly(InputStream in, int count) throws IOException { + byte[] out = new byte[count]; + int filled = 0; + while (filled < count) { + int n = in.read(out, filled, count - filled); + if (n < 0) { + return null; + } + filled += n; + } + return out; + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + private int countPets() throws Exception { + String listed = body(request("GET", "/pets", null, null)); + return countOccurrences(listed, "\"id\":"); + } + + private static int countOccurrences(String haystack, String needle) { + int count = 0; + int at = 0; + while ((at = haystack.indexOf(needle, at)) >= 0) { + count++; + at += needle.length(); + } + return count; + } + + /** One request on its own connection, returning the whole raw response. */ + private byte[] request(String method, String target, String body, String[] extraHeaders) + throws IOException { + StringBuilder head = new StringBuilder(); + head.append(method).append(' ').append(target).append(" HTTP/1.1\r\n"); + head.append("Host: 127.0.0.1\r\n"); + if (extraHeaders != null) { + for (String h : extraHeaders) { + head.append(h).append("\r\n"); + } + } + byte[] payload = body == null ? new byte[0] : body.getBytes(StandardCharsets.UTF_8); + head.append("Content-Length: ").append(payload.length).append("\r\n"); + head.append("Connection: close\r\n\r\n"); + return raw(head.toString(), payload); + } + + private byte[] raw(String head) throws IOException { + return raw(head, new byte[0]); + } + + private byte[] raw(String head, byte[] body) throws IOException { + return rawOn(port, head, body); + } + + /** Writes exactly these bytes, for a request whose body is not text. */ + private byte[] rawBytes(byte[] all) throws IOException { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", port), 5000); + socket.setSoTimeout(15000); + try { + socket.getOutputStream().write(all); + socket.getOutputStream().flush(); + return readFullyBytes(socket.getInputStream()); + } finally { + socket.close(); + } + } + + private byte[] rawOn(int onPort, String head, byte[] body) throws IOException { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress("127.0.0.1", onPort), 5000); + socket.setSoTimeout(15000); + try { + OutputStream out = socket.getOutputStream(); + try { + out.write(head.getBytes(StandardCharsets.UTF_8)); + if (body.length > 0) { + out.write(body); + } + out.flush(); + } catch (IOException earlyClose) { + // A server is allowed to answer and close before the body finishes + // arriving -- a 413 or a 503 is exactly that -- and then the rest + // of the write meets a closed socket. Reading its answer here is + // the difference between a test that reports "503" and one that + // reports "Broken pipe" and hides the reason. + byte[] answered = readFullyBytes(socket.getInputStream()); + if (answered.length > 0) { + return answered; + } + throw earlyClose; + } + return readFullyBytes(socket.getInputStream()); + } finally { + socket.close(); + } + } + + private static int status(byte[] response) { + return statusOf(response); + } + + private static int statusOf(byte[] response) { + String text = new String(response, StandardCharsets.UTF_8); + int firstSpace = text.indexOf(' '); + if (firstSpace < 0) { + return -1; + } + int secondSpace = text.indexOf(' ', firstSpace + 1); + try { + return Integer.parseInt(text.substring(firstSpace + 1, + secondSpace < 0 ? text.length() : secondSpace).trim()); + } catch (NumberFormatException err) { + return -1; + } + } + + private static String header(byte[] response, String name) { + String text = new String(response, StandardCharsets.UTF_8); + int end = text.indexOf("\r\n\r\n"); + String head = end < 0 ? text : text.substring(0, end); + for (String line : head.split("\r\n")) { + int colon = line.indexOf(':'); + if (colon > 0 && line.substring(0, colon).trim().equalsIgnoreCase(name)) { + return line.substring(colon + 1).trim(); + } + } + return null; + } + + private static String body(byte[] response) { + return new String(bodyBytes(response), StandardCharsets.UTF_8).trim(); + } + + private static byte[] bodyBytes(byte[] response) { + String text = new String(response, StandardCharsets.UTF_8); + int end = text.indexOf("\r\n\r\n"); + if (end < 0) { + return new byte[0]; + } + int start = end + 4; + byte[] out = new byte[response.length - start]; + System.arraycopy(response, start, out, 0, out.length); + return out; + } + + private static byte[] readFullyBytes(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int n; + try { + while ((n = in.read(buffer)) > 0) { + out.write(buffer, 0, n); + } + } catch (IOException err) { + // A read timeout means the peer said nothing more; whatever arrived is + // the response. + } + return out.toByteArray(); + } + + private static String readFully(InputStream in) throws IOException { + return new String(readFullyBytes(in), StandardCharsets.UTF_8); + } + + private static int freePort() throws IOException { + ServerSocket probe = new ServerSocket(0); + try { + return probe.getLocalPort(); + } finally { + probe.close(); + } + } + + private static boolean waitForPort(int port, long timeoutMillis) { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + Socket socket = new Socket(); + try { + socket.connect(new InetSocketAddress("127.0.0.1", port), 500); + return true; + } catch (IOException err) { + try { + Thread.sleep(100); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } + } finally { + try { + socket.close(); + } catch (IOException ignored) { + // closing a probe socket that never connected + } + } + } + return false; + } + + private static Path findJdk8() { + String env = System.getenv("JDK_8_HOME"); + if (env != null && Files.isExecutable(Paths.get(env, "bin", "javac"))) { + return Paths.get(env); + } + List candidates = new ArrayList(); + String home = System.getProperty("user.home"); + candidates.add(Paths.get("/Library/Java/JavaVirtualMachines")); + candidates.add(Paths.get(home, "Library", "Java", "JavaVirtualMachines")); + candidates.add(Paths.get("/usr/lib/jvm")); + for (Path root : candidates) { + if (!Files.isDirectory(root)) { + continue; + } + try { + java.util.Iterator it = Files.list(root).iterator(); + while (it.hasNext()) { + Path entry = it.next(); + String name = entry.getFileName().toString().toLowerCase(); + if (name.indexOf("1.8") < 0 && name.indexOf("-8") < 0 && name.indexOf("jdk8") < 0) { + continue; + } + Path javac = entry.resolve("Contents/Home/bin/javac"); + if (Files.isExecutable(javac)) { + return entry.resolve("Contents/Home"); + } + javac = entry.resolve("bin/javac"); + if (Files.isExecutable(javac)) { + return entry; + } + } + } catch (IOException err) { + // unreadable directory; try the next candidate + } + } + return null; + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendJavaSeRuntimeTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendJavaSeRuntimeTest.java new file mode 100644 index 00000000000..269210b3c01 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BackendJavaSeRuntimeTest.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The same runtime self-test, run against the LOCAL Java SE arm of the backend. + * + * vm/backend/src is one copy of the protocol logic and vm/backend/impl holds the + * two implementations under it -- natives for the translated target, JDK classes + * for the local dev loop. A dev loop that behaves differently from production is + * worse than no dev loop, so the same assertions run on both, and the counts have + * to match: {@link BackendRuntimeSelfTest} is the translated half of this pair. + */ +class BackendJavaSeRuntimeTest { + + @Test + @DisplayName("the local Java SE runtime passes the same checks as the translated one") + void javaSeSelfTest() throws Exception { + if (CompilerHelper.isWindows()) { + Assumptions.abort("the server-side backend is POSIX-only for now"); + } + BackendTestSupport.require(Files.isDirectory(BackendTestSupport.backendDir()), + "vm/backend is not present"); + + Path work = Files.createTempDirectory("backend-javase-selftest"); + Map env = new HashMap(); + // Not needed to RUN the local arm -- it is JDK classes all the way down -- + // but run-javase.sh generates the shared contract when it is missing, and + // that goes through maven and a JDK 8. Passed when there is one; the local + // loop still works without it once gen/ exists. + Path jdk8 = BackendTestSupport.findJdk8(); + if (jdk8 != null) { + env.put("JDK_8_HOME", jdk8.toString()); + } + // The test's own JVM, so this does not depend on what is on PATH. + env.put("CN1_BACKEND_JAVA", System.getProperty("java.home")); + env.put("CN1_BACKEND_DEMO", "demo/selftest"); + env.put("CN1_BACKEND_JDBC_JARS", BackendTestSupport.jdbcJars()); + // A real file: an in-memory database cannot be pooled, since every + // connection would get its own. + env.put("CN1_SELFTEST_DB", work.resolve("pool.db").toString()); + if (System.getenv("CN1_SELFTEST_NETWORK") != null) { + env.put("CN1_SELFTEST_NETWORK", "1"); + } + + List command = new ArrayList(Arrays.asList( + "./run-javase.sh", "com.demo.SelfTest")); + int[] status = new int[1]; + String output = BackendTestSupport.runBackendScript(command, env, 600, status); + + // generate-contract.sh needs codenameone-core and the maven plugin in the + // local repository, and says so by name rather than letting maven fail on an + // artifact nobody asked for. A job that has not installed them has an + // environment gap rather than a broken runtime, so it is skipped the same way + // the database tests skip without their service containers -- and + // CN1_BACKEND_REQUIRED still turns that skip into a failure where the backend + // is meant to be exercised. + if (output.indexOf("Install it first:") >= 0) { + BackendTestSupport.skipOrFail( + "the local Maven repository has no codenameone-core to build the " + + "contract against:\n" + output); + } + + assertTrue(output.indexOf("SELFTEST OK") >= 0, + "the local Java SE runtime reported failures:\n" + output); + assertEquals(0, status[0], output); + int passed = passedCount(output); + assertTrue(passed >= 80, + "expected the full set of checks, only " + passed + " ran:\n" + output); + } + + private static int passedCount(String output) { + int at = output.indexOf("passed="); + if (at < 0) { + return -1; + } + int end = output.indexOf(' ', at); + try { + return Integer.parseInt(output.substring(at + "passed=".length(), + end < 0 ? output.length() : end).trim()); + } catch (NumberFormatException err) { + return -1; + } + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendRuntimeSelfTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendRuntimeSelfTest.java new file mode 100644 index 00000000000..10b56e6b0e4 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BackendRuntimeSelfTest.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The runtime self-test, run against the TRANSLATED arm of the backend. + * + * The Java SE half of this pair, {@link BackendJavaSeRuntimeTest}, has always + * described itself as one of two -- but the class it named did not exist, so + * every one of these assertions ran only on the JVM. That is the arm that does + * NOT ship: the natives under vm/backend/impl/parparvm, the GC, and the buffer + * handling are exactly what a JDK-classes run cannot speak for. This is the + * other half. + */ +class BackendRuntimeSelfTest { + + @Test + @DisplayName("the translated runtime passes the same checks as the Java SE one") + void translatedSelfTest() throws Exception { + if (CompilerHelper.isWindows()) { + BackendTestSupport.skipOrFail("the server-side backend is POSIX-only for now"); + } + Path backend = Paths.get("..", "backend").normalize().toAbsolutePath(); + BackendTestSupport.require(Files.isDirectory(backend), "vm/backend is not present"); + Path jdk8 = BackendTestSupport.findJdk8(); + BackendTestSupport.require(jdk8 != null, "no JDK 8 available to compile the backend"); + + Path work = Files.createTempDirectory("backend-translated-selftest"); + Path binary = work.resolve("selftest"); + + // The real build script, for the same reason the other native tests use + // it: a test that builds differently from the product tests something + // else. + ProcessBuilder build = new ProcessBuilder("./build.sh", "SelfTest", "com.demo", + binary.toString()); + build.directory(backend.toFile()); + build.environment().put("JDK_8_HOME", jdk8.toString()); + build.environment().put("JAVA_HOME", jdk8.toString()); + build.environment().put("CN1_BACKEND_DEMO", "demo/selftest"); + build.redirectErrorStream(true); + Process built = build.start(); + String buildLog = readAll(built); + if (!built.waitFor(20, TimeUnit.MINUTES) || built.exitValue() != 0 + || !Files.isExecutable(binary)) { + BackendTestSupport.skipOrFail("could not build the self-test binary:\n" + + tail(buildLog)); + } + + ProcessBuilder run = new ProcessBuilder(binary.toString()); + // A real file: an in-memory database cannot be pooled, because every + // connection would get one of its own. + run.environment().put("CN1_SELFTEST_DB", work.resolve("pool.db").toString()); + if (System.getenv("CN1_SELFTEST_NETWORK") != null) { + run.environment().put("CN1_SELFTEST_NETWORK", "1"); + } + run.redirectErrorStream(true); + Process p = run.start(); + String output = readAll(p); + boolean ended = p.waitFor(10, TimeUnit.MINUTES); + if (!ended) { + p.destroyForcibly(); + } + assertTrue(ended, "the self-test never finished:\n" + tail(output)); + assertTrue(output.indexOf("SELFTEST OK") >= 0, + "the translated runtime failed its own checks:\n" + tail(output)); + // The same floor the Java SE half asserts, so a run that quietly stopped + // early cannot pass by printing OK after three checks. + int passed = passedCount(output); + assertTrue(passed >= 80, + "expected the full set of checks, only " + passed + " ran:\n" + tail(output)); + } + + /** Reads "passed=N" out of the self-test's own summary line. */ + private static int passedCount(String output) { + int at = output.indexOf("passed="); + if (at < 0) { + return 0; + } + int end = at + "passed=".length(); + while (end < output.length() && Character.isDigit(output.charAt(end))) { + end++; + } + try { + return Integer.parseInt(output.substring(at + "passed=".length(), end)); + } catch (NumberFormatException notANumber) { + return 0; + } + } + + private static String readAll(Process p) throws java.io.IOException { + java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + java.io.InputStream in = p.getInputStream(); + for (;;) { + int n = in.read(buffer); + if (n < 0) { + break; + } + out.write(buffer, 0, n); + } + return new String(out.toByteArray(), "UTF-8"); + } + + private static String tail(String text) { + return text.length() > 3000 ? text.substring(text.length() - 3000) : text; + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendTestSupport.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendTestSupport.java new file mode 100644 index 00000000000..0746b406811 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BackendTestSupport.java @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Shared plumbing for the tests that drive a translated server binary: find a JDK + * 8, build vm/backend with its own build script, and wait for the port. + * + * The build goes through the real build.sh rather than a reimplementation of it. A + * test that builds differently from the product is testing something else. + */ +final class BackendTestSupport { + + private BackendTestSupport() { + } + + /** + * Set CN1_BACKEND_REQUIRED=1 where the backend is expected to build -- CI, in + * particular. Without it a missing toolchain skips these tests, which is right + * on a developer machine and wrong on a build machine: a suite that quietly + * stops running is worse than no suite, because it still reports green. + */ + static boolean isRequired() { + return "1".equals(System.getenv("CN1_BACKEND_REQUIRED")); + } + + /** + * Skips, or fails when the backend is required here. Every abort in these + * tests goes through this so no single one can be forgotten. + */ + static void skipOrFail(String reason) { + if (isRequired()) { + org.junit.jupiter.api.Assertions.fail( + "CN1_BACKEND_REQUIRED is set, so this must not be skipped: " + reason); + } + org.junit.jupiter.api.Assumptions.abort(reason); + } + + /** The assume/abort pair, routed through skipOrFail. */ + static void require(boolean condition, String reason) { + if (!condition) { + skipOrFail(reason); + } + } + + static Path backendDir() { + return Paths.get("..", "backend").normalize().toAbsolutePath(); + } + + /** Builds one demo into `binary`. Returns null on success, or the reason. */ + static String build(String mainClass, String demoDir, Path binary, Path jdk8) throws Exception { + ProcessBuilder build = new ProcessBuilder("./build.sh", mainClass, "com.demo", + binary.toString()); + build.directory(backendDir().toFile()); + build.environment().put("JDK_8_HOME", jdk8.toString()); + build.environment().put("JAVA_HOME", jdk8.toString()); + build.environment().put("CN1_BACKEND_DEMO", demoDir); + build.redirectErrorStream(true); + Process p = build.start(); + String log = readFully(p.getInputStream()); + boolean ok = p.waitFor(20, TimeUnit.MINUTES) && p.exitValue() == 0 + && Files.isExecutable(binary); + if (ok) { + return null; + } + return "could not build " + mainClass + ":\n" + + (log.length() > 3000 ? log.substring(log.length() - 3000) : log); + } + + static Process start(Path binary, Map env, Path logFile) throws IOException { + ProcessBuilder run = new ProcessBuilder(binary.toString()); + run.environment().putAll(env); + run.redirectErrorStream(true); + run.redirectOutput(logFile.toFile()); + return run.start(); + } + + static void stop(Process server) { + if (server == null) { + return; + } + server.destroy(); + try { + if (!server.waitFor(10, TimeUnit.SECONDS)) { + server.destroyForcibly(); + } + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + } + + static int freePort() throws IOException { + ServerSocket probe = new ServerSocket(0); + try { + return probe.getLocalPort(); + } finally { + probe.close(); + } + } + + static boolean waitForPort(int port, long timeoutMillis) { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + Socket socket = new Socket(); + try { + socket.connect(new InetSocketAddress("127.0.0.1", port), 500); + return true; + } catch (IOException err) { + try { + Thread.sleep(100); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return false; + } + } finally { + try { + socket.close(); + } catch (IOException ignored) { + // closing a probe socket that never connected + } + } + } + return false; + } + + /** + * The JDBC jars on this test run's own classpath, as a path list. + * + * The local Java SE arm of the backend reaches SQLite through a driver, and + * hunting for one in ~/.m2 makes a test depend on whatever some other build + * happened to leave there. These are declared dependencies of this module, so + * they are the same jars on every machine. + */ + static String jdbcJars() { + StringBuilder out = new StringBuilder(); + String separator = System.getProperty("path.separator", ":"); + String[] entries = System.getProperty("java.class.path", "").split(java.util.regex.Pattern.quote(separator)); + for (String entry : entries) { + String name = new java.io.File(entry).getName(); + if (name.startsWith("sqlite-jdbc") || name.startsWith("slf4j-api")) { + if (out.length() > 0) { + out.append(separator); + } + out.append(entry); + } + } + return out.toString(); + } + + /** + * Runs one of vm/backend's own scripts, with the environment those scripts + * read already filled in. Returns the combined output; `exitCode` is written + * into `status[0]` so a caller can tell a failed run from a quiet one. + */ + static String runBackendScript(List command, Map env, + long timeoutSeconds, int[] status) throws Exception { + ProcessBuilder pb = new ProcessBuilder(command); + pb.directory(backendDir().toFile()); + pb.environment().putAll(env); + pb.redirectErrorStream(true); + Process p = pb.start(); + String out = readFully(p.getInputStream()); + if (!p.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + p.destroyForcibly(); + status[0] = -1; + return out; + } + status[0] = p.exitValue(); + return out; + } + + /** Whether a command exists on PATH, so a test can skip rather than fail. */ + static boolean hasCommand(String command) { + try { + ProcessBuilder pb = new ProcessBuilder(command, "--version"); + pb.redirectErrorStream(true); + Process p = pb.start(); + readFully(p.getInputStream()); + return p.waitFor(20, TimeUnit.SECONDS) && p.exitValue() == 0; + } catch (Exception err) { + return false; + } + } + + /** Runs a command and returns its combined output, or null when it failed. */ + static String run(List command, long timeoutSeconds) { + try { + ProcessBuilder pb = new ProcessBuilder(command); + pb.redirectErrorStream(true); + Process p = pb.start(); + String out = readFully(p.getInputStream()); + if (!p.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + p.destroyForcibly(); + return null; + } + return p.exitValue() == 0 ? out : null; + } catch (Exception err) { + return null; + } + } + + static String readFully(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int n; + try { + while ((n = in.read(buffer)) > 0) { + out.write(buffer, 0, n); + } + } catch (IOException err) { + // a read timeout means the peer said nothing more + } + return new String(out.toByteArray(), StandardCharsets.UTF_8); + } + + static Path findJdk8() { + String env = System.getenv("JDK_8_HOME"); + if (env != null && Files.isExecutable(Paths.get(env, "bin", "javac"))) { + return Paths.get(env); + } + List roots = new ArrayList(); + String home = System.getProperty("user.home"); + roots.add(Paths.get("/Library/Java/JavaVirtualMachines")); + roots.add(Paths.get(home, "Library", "Java", "JavaVirtualMachines")); + roots.add(Paths.get("/usr/lib/jvm")); + for (Path root : roots) { + if (!Files.isDirectory(root)) { + continue; + } + try { + java.util.Iterator it = Files.list(root).iterator(); + while (it.hasNext()) { + Path entry = it.next(); + String name = entry.getFileName().toString().toLowerCase(); + if (name.indexOf("1.8") < 0 && name.indexOf("-8") < 0 && name.indexOf("jdk8") < 0) { + continue; + } + if (Files.isExecutable(entry.resolve("Contents/Home/bin/javac"))) { + return entry.resolve("Contents/Home"); + } + if (Files.isExecutable(entry.resolve("bin/javac"))) { + return entry; + } + } + } catch (IOException err) { + // unreadable directory; try the next root + } + } + return null; + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java index a602239f7a3..c32d6c1ce2a 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java @@ -251,6 +251,15 @@ private void runFloorProbe(List tempDirs) throws Exception { m.containsKey("RELEASED") ? m.get("RELEASED") : -1, m.containsKey("MINAFTER") ? m.get("MINAFTER") : -1)); } + // How much of the release wait was spent. Sized for the SLOWEST marker + // configuration in the matrix, so the margin has to be visible: a green + // run that used its whole budget is about to go red on a slower runner. + Matcher settle = Pattern.compile("ARM_SETTLE name=(\\S+) rounds=(\\d+) maxRounds=(\\d+)") + .matcher(vmOutput); + while (settle.find()) { + report.append(String.format("%-24s settle used %s of %s rounds%n", + settle.group(1), settle.group(2), settle.group(3))); + } System.err.println("[BibopPageFloorIntegrationTest] texture set " + TEXTURE_SET_KB + "KB, phys_footprint\n" + report); @@ -265,6 +274,19 @@ private void runFloorProbe(List tempDirs) throws Exception { // phase; so does a run where task_info was unavailable. Nothing can be // measured then, and failing would report a porting/environment gap as a // memory regression. + // Some arm64 hosts run a 64KB system page, and on those the collector + // DECLINES to release pages at all: the rounded page header plus one + // system page no longer fits inside a 64KB BiBOP page, so the release + // offset is zero and the whole process keeps its footprint by design. + // The floor this test measures cannot exist there, and failing would + // report a host property as a collector regression. The runtime says so + // itself rather than this test guessing -- so if the line is absent the + // failure is real and still fails. + org.junit.jupiter.api.Assumptions.assumeFalse( + lastVmStderr.indexOf("page release unavailable on this host") >= 0, + "this host cannot release pages, so there is no floor to measure\n" + + report); + org.junit.jupiter.api.Assumptions.assumeTrue(warmupHeld > 0, "This run could not read phys_footprint through Runtime, so the probe cannot be " + "measured here.\n" + report); @@ -409,18 +431,31 @@ private String runVm(Path executable, Path workingDir) throws Exception { // or not, since cn1BibopTrimFreePool runs only at the end of one, so a // failing run has to say whether one ran and what it spliced. builder.environment().put("CN1_LOG_PAGE_RELEASE", "1"); - builder.redirectError(ProcessBuilder.Redirect.INHERIT); + // To a FILE rather than INHERIT, for the same reason it is not merged: + // the stream stays separate so nothing splices a marker line, but the + // test can now READ it. The collector reports a host that cannot release + // pages at all on stderr, and a diagnosis this test has to act on is no + // use if only a human scrolling the log can see it. It is echoed below + // so the CI log keeps exactly what INHERIT used to show. + Path errFile = workingDir.resolve("vm-stderr.txt"); + builder.redirectError(ProcessBuilder.Redirect.to(errFile.toFile())); Process process = builder.start(); String output; try (BufferedReader reader = new BufferedReader( new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { output = reader.lines().collect(Collectors.joining("\n")); } - assertEquals(0, process.waitFor(), - "ParparVM run should exit cleanly. Output: " + output); + int exit = process.waitFor(); + lastVmStderr = Files.exists(errFile) + ? new String(Files.readAllBytes(errFile), StandardCharsets.UTF_8) : ""; + System.err.print(lastVmStderr); + assertEquals(0, exit, "ParparVM run should exit cleanly. Output: " + output); return output; } + /** Whatever the last translated run wrote to stderr; see runVm. */ + private String lastVmStderr = ""; + private String loadAppSource() throws Exception { java.io.InputStream in = BibopPageFloorIntegrationTest.class .getResourceAsStream("/com/codename1/tools/translator/BibopPageFloorApp.java"); diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/BibopPageFloorApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/BibopPageFloorApp.java index b18f0f767e6..3731a3a85ee 100644 --- a/vm/tests/src/test/resources/com/codename1/tools/translator/BibopPageFloorApp.java +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/BibopPageFloorApp.java @@ -115,9 +115,28 @@ public class BibopPageFloorApp { * "the collector had not finished yet". System.gc() is asynchronous (it sets * forceGc and notifies the collector thread, then returns), so each round is * a request plus a pause long enough for a full cycle to land. + * + *

The ceiling is generous because the wait is not the interesting part, + * and raising it is NOT a fix for the single-marker failures on arm64 CI: + * it was raised for those and they continued. What the reported round count + * then showed is that the behaviour is bimodal rather than slow. Two + * adjacent commits with identical collector code, same 1-marker job: one + * released inside four rounds (one second), the other spent all 80 (twenty + * seconds) and gave back NOTHING, and the memory then came back during the + * next phase instead. A budget cannot fix "never" -- something is holding + * the warm-up's live set across the settle, and the next phase's frames + * overwriting it is what lets go. See the scrubStack note below, which + * records the same behaviour from the first time it was seen. + * + *

Note the ceiling cannot be replaced by "stop once the footprint stops + * falling". In the failing runs the footprint is flat for the whole window, + * so a flatness rule gives up sooner and reports the same wrong answer -- + * the trap SETTLE_STABLE_STREAK below already exists to avoid. Only an + * absolute budget works here, and it stays finite so a release that never + * comes still fails the assertion rather than hanging. */ private static final int SETTLE_MIN_ROUNDS = 4; - private static final int SETTLE_MAX_ROUNDS = 20; + private static final int SETTLE_MAX_ROUNDS = 80; private static final int SETTLE_PLAIN_MIN_ROUNDS = 4; private static final int SETTLE_PLAIN_MAX_ROUNDS = 12; private static final long SETTLE_PAUSE_MS = 250; @@ -286,7 +305,12 @@ private static void endPhase(String name, String stats) { private static void releasePhase(String name, long heldKb, boolean expectDrop) { scrubStack(SCRUB_DEPTH); if (expectDrop) { - settleForRelease(heldKb); + // Report what the wait actually cost. A run that passes while + // spending its whole budget is one runner away from failing, and + // that is invisible if only the outcome is printed. + System.out.println("ARM_SETTLE name=" + name + + " rounds=" + settleForRelease(heldKb) + + " maxRounds=" + SETTLE_MAX_ROUNDS); } else { settle(); } @@ -357,15 +381,16 @@ private static long hold(byte[][] live, int elementSize) { * fails the assertion -- it just fails on the real behaviour rather than on * whichever machine ran it. */ - private static void settleForRelease(long heldKb) { + private static int settleForRelease(long heldKb) { long target = (heldKb * 3) / 5; for (int i = 0; i < SETTLE_MAX_ROUNDS; i++) { System.gc(); sleep(SETTLE_PAUSE_MS); if (i + 1 >= SETTLE_MIN_ROUNDS && footprintKb() <= target) { - return; + return i + 1; } } + return SETTLE_MAX_ROUNDS; } private static long scrubStack(int depth) {