benchmark.py

"""Measures the four performance claims the handbook makes.

Every number the handbook prints comes from this script, run against the
generated dataset. Nothing is estimated. If a claim cannot be measured, it is
not made.

    python3 generate_dataset.py     # first
    python3 benchmark.py

Environment: NEO4J_URI, NEO4J_USER, NEO4J_PASSWORD.
"""

import os
import statistics
import time
from neo4j import GraphDatabase

URI = os.environ.get("NEO4J_URI", "bolt://localhost:7687")
AUTH = (os.environ.get("NEO4J_USER", "neo4j"), os.environ["NEO4J_PASSWORD"])
DB = os.environ.get("NEO4J_DATABASE", "neo4j")


def q(d, cy, **kw):
    return d.execute_query(cy, database_=DB, **kw)


def timed(fn, runs=5):
    """Median of N runs, after a warm-up, in milliseconds."""
    fn()
    ts = []
    for _ in range(runs):
        t = time.perf_counter()
        fn()
        ts.append((time.perf_counter() - t) * 1000)
    return statistics.median(ts)


def db_hits(d, cy, **kw):
    """Total database accesses reported by PROFILE."""
    _, s, _ = d.execute_query("PROFILE " + cy, database_=DB, **kw)
    total, stack = 0, [s.profile]
    while stack:
        p = stack.pop()
        total += p.get("dbHits", 0)
        stack.extend(p.get("children", []))
    return total


def main():
    with GraphDatabase.driver(URI, auth=AUTH) as d:
        d.verify_connectivity()
        n = q(d, "MATCH (n) RETURN count(n) AS c")[0][0]["c"]
        r = q(d, "MATCH ()-[x]->() RETURN count(x) AS c")[0][0]["c"]
        print(f"dataset: {n:,} nodes, {r:,} relationships\n")

        # ── 1. the index, measured as database hits ──────────────────────
        print("1. LOOKUP WITH AND WITHOUT AN INDEX")
        LOOKUP = "MATCH (e:Engineer {email:$email}) RETURN e.name AS n"
        email = "eng25000@example.com"
        with_idx = db_hits(d, LOOKUP, email=email)
        t_with = timed(lambda: q(d, LOOKUP, email=email))

        q(d, "DROP CONSTRAINT engineer_email IF EXISTS")
        time.sleep(2)
        without_idx = db_hits(d, LOOKUP, email=email)
        t_without = timed(lambda: q(d, LOOKUP, email=email))
        q(d, "CREATE CONSTRAINT engineer_email IF NOT EXISTS FOR (e:Engineer) REQUIRE e.email IS UNIQUE")
        time.sleep(3)

        print(f"   without index : {without_idx:>8,} db hits   {t_without:7.1f} ms")
        print(f"   with index    : {with_idx:>8,} db hits   {t_with:7.1f} ms")
        print(f"   ratio         : {without_idx/max(with_idx,1):>8.0f}x fewer hits, "
              f"{t_without/max(t_with,0.01):.0f}x faster\n")

        # ── 2. round trips: one at a time versus UNWIND ──────────────────
        print("2. WRITING 1,000 ROWS, ONE AT A TIME VERSUS UNWIND")
        rows = [{"email": f"bench{i}@example.com", "name": f"Bench {i}"} for i in range(1000)]
        q(d, "MATCH (e:Engineer) WHERE e.email STARTS WITH 'bench' DETACH DELETE e")

        t = time.perf_counter()
        for row in rows:
            q(d, "MERGE (e:Engineer {email:$email}) SET e.name=$name", **row)
        one_at_a_time = (time.perf_counter() - t) * 1000
        q(d, "MATCH (e:Engineer) WHERE e.email STARTS WITH 'bench' DETACH DELETE e")

        t = time.perf_counter()
        q(d, """UNWIND $rows AS row
                MERGE (e:Engineer {email:row.email}) SET e.name=row.name""", rows=rows)
        unwind = (time.perf_counter() - t) * 1000
        q(d, "MATCH (e:Engineer) WHERE e.email STARTS WITH 'bench' DETACH DELETE e")

        print(f"   one statement per row : {one_at_a_time:8.0f} ms  (1,000 round trips)")
        print(f"   one UNWIND            : {unwind:8.0f} ms  (1 round trip)")
        print(f"   ratio                 : {one_at_a_time/max(unwind,0.01):8.0f}x faster\n")

        # ── 3. the multi-hop query on a real graph ───────────────────────
        # ⚠ PICK THE START NODE DETERMINISTICALLY. This used to be `LIMIT 1`
        # with no ORDER BY, which returns whichever node the storage engine
        # happens to hand back first. That is not stable across reloads, so two
        # people running this got different answers from the same dataset. A
        # fresh-clone test caught it: section 4 returned 3, 9 and 27 services on
        # one machine and 0, 0, 0 on the next.
        print("3. THE MULTI-HOP QUERY")
        ref = q(d, "MATCH (i:Incident) RETURN i.ref AS r ORDER BY r LIMIT 1")[0][0]["r"]
        MULTI = """MATCH (i:Incident {ref:$ref})-[:AFFECTS]->(:Service)<-[:OWNS]-(e:Engineer)
                   RETURN DISTINCT e.name AS name"""
        rows_out = len(q(d, MULTI, ref=ref)[0])
        print(f"   {rows_out} engineers, {db_hits(d, MULTI, ref=ref):,} db hits, "
              f"{timed(lambda: q(d, MULTI, ref=ref)):.1f} ms")
        print("   the graph is 5,000x larger than the teaching dataset and this is unchanged\n")

        # ── 4. bounded versus unbounded variable length ──────────────────
        print("4. VARIABLE LENGTH PATH, BOUNDED VERSUS UNBOUNDED")
        # Start from the most depended-upon service, chosen deterministically:
        # an arbitrary service may have no incoming DEPENDS_ON at all, in which
        # case every bound reaches nothing and the demonstration shows nothing.
        # Ties broken by name so the choice is identical on every machine.
        svc = q(d, """MATCH (s:Service)<-[:DEPENDS_ON]-()
                      WITH s, count(*) AS c
                      RETURN s.name AS n ORDER BY c DESC, n ASC LIMIT 1""")[0][0]["n"]
        for bound in ("1..2", "1..4", "1..6"):
            cy = f"MATCH (s:Service {{name:$n}})<-[:DEPENDS_ON*{bound}]-(x:Service) RETURN count(DISTINCT x) AS c"
            reached = q(d, cy, n=svc)[0][0]["c"]
            print(f"   *{bound:<5} reached {reached:>6,} services, "
                  f"{db_hits(d, cy, n=svc):>9,} db hits, {timed(lambda: q(d, cy, n=svc), 3):7.1f} ms")


if __name__ == "__main__":
    main()