diff --git a/2.0/problems/bboplace_direct_iccad2015/config.yaml b/2.0/problems/bboplace_direct_iccad2015/config.yaml index de16070a..fff8248b 100644 --- a/2.0/problems/bboplace_direct_iccad2015/config.yaml +++ b/2.0/problems/bboplace_direct_iccad2015/config.yaml @@ -2,17 +2,20 @@ tag: optimization runtime: language: json timeout_seconds: 10800 - environment: "JSON placement for one hidden ICCAD2015 BBOPlace design" + environment: "JSON placement for one visible ICCAD2015 BBOPlace design; hidden evaluator" apt_packages: - python3-numpy docker: image: ubuntu:24.04 judge_image: ghcr.io/frontiercs/frontiercs-bboplace-data:2026-06-ispd-iccad + visible_inputs: + - source: /opt/bboplace-bench/benchmarks/iccad2015/superblue1 + destination: /app/input/benchmarks/iccad2015/superblue1 submission: kind: file path: /app/solution.json environment: - cpus: 8 + cpus: 4 memory_mb: 16384 storage_mb: 8192 build_timeout_seconds: 3600 diff --git a/2.0/problems/bboplace_direct_iccad2015/harbor/app/input_manifest.json b/2.0/problems/bboplace_direct_iccad2015/harbor/app/input_manifest.json new file mode 100644 index 00000000..a90e8f5e --- /dev/null +++ b/2.0/problems/bboplace_direct_iccad2015/harbor/app/input_manifest.json @@ -0,0 +1,88 @@ +{ + "dataset": "iccad2015", + "benchmark": "superblue1", + "benchmark_path": "/app/input/benchmarks/iccad2015/superblue1", + "placer": "mgo", + "metric": "mp_hpwl", + "objective": "minimize", + "dim": 1024, + "node_cnt": 512, + "net_cnt": 325, + "canvas_width": 8543.16, + "canvas_height": 3131.01, + "canvas_lx": 0.0, + "canvas_ly": 0.0, + "n_grid_x": 224, + "n_grid_y": 224, + "bounds_kind": "mgo_repeated_grid", + "max_candidates_per_submission": 1, + "baseline_hpwl": 69600.0, + "files": [ + "superblue1.def", + "superblue1.iccad2015", + "superblue1.lef", + "superblue1.sdc", + "superblue1.v", + "superblue1_Early.lib", + "superblue1_Late.lib" + ], + "submission_path": "/app/solution.json", + "coordinate_format": "placement[0:node_cnt] are x-grid coordinates; placement[node_cnt:2*node_cnt] are y-grid coordinates", + "macro_selection": { + "source_files": [ + "superblue1.def", + "superblue1.lef" + ], + "selected_nodes": "largest DEF components by LEF width*height area", + "count": 512, + "order": "descending width*height area; ties keep DEF component order" + }, + "net_selection": { + "source_file": "superblue1.v", + "count": 325, + "rules": [ + "scan Verilog wire declarations to collect valid net names", + "scan instance connections after the Start cells marker", + "keep only instance pins whose instance name is one of the 512 selected macros", + "use LEF pin rectangle centers as lower-left based pin offsets", + "convert kept pin offsets to center-relative offsets by subtracting half of that selected macro's width and height", + "discard nets with fewer than two selected macros", + "preserve the remaining net order from superblue1.v" + ], + "degree_histogram": { + "2": 225, + "3": 47, + "4": 18, + "5": 28, + "6": 2, + "9": 2, + "13": 3 + } + }, + "mgo_decode": { + "grid": [224, 224], + "grid_cell_size": { + "width": 38.13910714285714, + "height": 13.977723214285716 + }, + "def_to_grid_hint": "Visible DEF coordinates are database-unit component positions. The evaluator rescales DEF positions by the FE_CORE_BOX dimensions and DIEAREA before comparing to the MGO canvas. To convert a scaled macro bottom-left coordinate to a requested MGO grid coordinate, use x_grid=(x-canvas_lx)/grid_cell_width and y_grid=(y-canvas_ly)/grid_cell_height.", + "coordinate_bounds": { + "x": [0, 224], + "y": [0, 224] + }, + "coordinate_semantics": "desired MGO grid coordinates; evaluator floors floating coordinates before grid lookup and may move macros during legality repair", + "legalizer_rank": { + "rank_key": "area_sum", + "definition": "For each selected net, net area is the sum of width*height over macros on that net. For each macro, area_sum is the sum of net areas over selected nets containing that macro. Macros are greedily legalized in descending area_sum order; ties keep macro order." + }, + "decode_summary": [ + "candidate[0:512] and candidate[512:1024] are floored to requested grid ids", + "scaled macro size is ceil(width/grid_cell_width), ceil(height/grid_cell_height)", + "for each macro in legalizer_rank order, all grid cells that would overlap already placed macros or exceed the legal grid are masked out", + "among legal cells, the evaluator minimizes incremental HPWL to nets that already have placed macros, using pin offsets from superblue1.lef and superblue1.v", + "ties in incremental HPWL are broken by the L1 physical distance from the requested floored grid coordinate", + "the final macro bottom-left position is grid_id_x*grid_cell_width, grid_id_y*grid_cell_height in the evaluator's canvas-local coordinate system" + ], + "hpwl_summary": "After legalization, MP-HPWL is the sum over the 325 selected nets of (max pin x - min pin x) + (max pin y - min pin y), where pin x/y is macro bottom-left plus half macro size plus the center-relative pin offset." + } +} diff --git a/2.0/problems/bboplace_direct_iccad2015/readme b/2.0/problems/bboplace_direct_iccad2015/readme index ce3a2b75..f411f98e 100644 --- a/2.0/problems/bboplace_direct_iccad2015/readme +++ b/2.0/problems/bboplace_direct_iccad2015/readme @@ -1,9 +1,70 @@ BBOPlace Direct ICCAD2015 ======================== -Directly submit one JSON macro placement for a single BBOPlace ICCAD2015 -design: `superblue1`. The hidden judge evaluates the placement with the -original BBOPlace-Bench MGO MP-HPWL evaluator. +Problem +------- + +Directly submit one JSON macro-placement vector for one visible ICCAD2015 +design: `superblue1`. The hidden judge evaluates your vector with the original +BBOPlace-Bench Mask-Guided Optimization (MGO) MP-HPWL evaluator. + +This is a VLSI chip-placement optimization task. A chip design contains many +rectangular objects. Large objects are called macros, smaller logic objects are +called standard cells, and nets describe which pins on those objects must be +connected by wires. A placement assigns physical locations to movable objects +on a rectangular chip canvas. Good placements keep connected objects near each +other while avoiding illegal macro overlap and boundary violations. + +This direct variant gives you the raw `superblue1` ICCAD2015 input files in the +agent workspace. You may parse those files, run local search, generate many +trial JSON files, and call the submit helper for feedback. Only the final +`/app/solution.json` artifact is graded. + +Your JSON vector is not a final DEF placement. It is a BBOPlace MGO genotype: +one desired grid coordinate for each selected macro. The hidden evaluator +decodes the vector by placing macros on a 2D grid, masking out illegal grid +positions, greedily adjusting macros to legal positions, and then reporting +macro-placement half-perimeter wirelength (MP-HPWL). Lower MP-HPWL is better. + +Background definitions +---------------------- + +A net connects two or more pins. The half-perimeter wirelength of one net is: + +```text +(max pin x - min pin x) + (max pin y - min pin y) +``` + +HPWL is the sum of that value over nets. MP-HPWL is the same idea restricted to +the macro-placement evaluation path used by BBOPlace-Bench. The judge reports +only MP-HPWL for scoring; it does not run global placement of standard cells. + +ICCAD2015 uses an LEF/DEF-style flow. For this task the visible files have +these roles: + +```text +.def design components, floorplan, rows, and initial placements +.lef macro geometry and pin shapes +.v logical netlist used to select macro-to-macro nets +.sdc timing constraints, not used by the MP-HPWL evaluator +*_Early.lib early timing library, not used by the MP-HPWL evaluator +*_Late.lib late timing library, not used by the MP-HPWL evaluator +``` + +Useful ICCAD parsing notes: + +- `.def` component coordinates are in database units. The official MGO canvas + constants for this task are also provided in `/app/input_manifest.json`. +- `.lef` contains the macro sizes and pin rectangles needed for a local + macro-only model. +- `.v` contains the logical instance-to-net connections used to build + macro-to-macro nets for MP-HPWL. +- `.sdc` and `.lib` files are present for completeness but are not needed by + the MP-HPWL scoring path. + +You do not need to reproduce the official HPWL or legalizer exactly to submit a +valid JSON. Local parsers are useful only for choosing better coordinates before +calling `/app/submit.sh`. Runtime and resources --------------------- @@ -11,61 +72,196 @@ Runtime and resources Your final submission is a JSON file at `/app/solution.json`. You are encouraged to write Python programs, shell scripts, search loops, local parsers, or any other helper code in `/app` while working. Those programs may generate and -overwrite `/app/solution.json` many times. Only the final JSON artifact is -graded. +overwrite `/app/solution.json` many times. The agent container provides: -- 8 CPU cores +- 4 CPU cores - 16 GiB memory - 8 GiB storage - no GPU - Python 3 with NumPy available for local helper scripts -- hidden benchmark data is not available in the agent workspace +- the `superblue1` benchmark input is available in the agent workspace at + `/app/input/benchmarks/iccad2015/superblue1` The judge also runs on CPU only. Do not rely on CUDA, DREAMPlace, Ray, or GPU placement libraries for scoring. +Visible input files +------------------- + +The agent can inspect the raw ICCAD2015 placement input while generating +`/app/solution.json`: + +```text +/app/input_manifest.json +/app/input/benchmarks/iccad2015/superblue1/superblue1.def +/app/input/benchmarks/iccad2015/superblue1/superblue1.iccad2015 +/app/input/benchmarks/iccad2015/superblue1/superblue1.lef +/app/input/benchmarks/iccad2015/superblue1/superblue1.v +/app/input/benchmarks/iccad2015/superblue1/superblue1.sdc +/app/input/benchmarks/iccad2015/superblue1/superblue1_Early.lib +/app/input/benchmarks/iccad2015/superblue1/superblue1_Late.lib +``` + +The evaluator implementation and scoring runtime remain hidden in the judge +image. + +`/app/input_manifest.json` is the machine-readable public data channel for +instance-specific values and evaluator-facing conventions. Read it instead of +hard-coding dimensions or inferring hidden details from feedback. It includes +the placement dimension, macro count, grid bounds, canvas information, scoring +baseline, macro-selection rule, selected-net rule, and a concise description of +the MGO decode/legalization behavior. Together with the visible LEF/DEF/Verilog +files, that manifest is intended to be sufficient for building a useful local +model; external repositories or internet search should not be needed. + Submission format ----------------- Submit exactly one placement for `superblue1` by writing `/app/solution.json`. -After your program writes the file, call `bash /app/submit.sh` to score that -JSON. The JSON must use one of these forms: +After your program writes the file, call: + +```bash +bash /app/submit.sh +``` + +to score that JSON through the black-box judge. + +The JSON must use one of these forms: ```json -{"placement": [0.0, 0.0, "..."]} +{"placement": [0.0, 0.0, 1.0, 1.0]} ``` or: ```json -{"x": [0.0, 0.0, "..."], "y": [0.0, 0.0, "..."]} +{"x": [0.0, 1.0], "y": [0.0, 1.0]} ``` -The placement vector length must equal `dim = 2 * node_cnt`. The first +For smoke testing, this shorthand is also accepted: + +```json +{"fill": 0.0} +``` + +`{"fill": v}` expands inside the evaluator to a full vector where every +coordinate is `v`. It is a convenient way to create a syntactically valid smoke +test submission. + +The placement vector length must equal `dim = 2 * node_cnt = 1024`. The first `node_cnt` entries are x-grid coordinates and the remaining `node_cnt` entries -are y-grid coordinates. Coordinates must be finite, with x in `[0, n_grid_x]` -and y in `[0, n_grid_y]`. Only one placement is accepted. +are y-grid coordinates: + +```text +placement[0:node_cnt] = x-grid coordinates +placement[node_cnt:2 * node_cnt] = y-grid coordinates +``` -The judge discloses public metadata through the iterative feedback path, -including `dim`, `node_cnt`, `n_grid_x`, `n_grid_y`, and the baseline HPWL. -The netlist and evaluator source stay hidden in the judge image. +Coordinates must be finite. X coordinates must be in `[0, n_grid_x]`; y +coordinates must be in `[0, n_grid_y]`. Coordinates may be integers or floats. +Only one placement is accepted. -Score ------ +The public metadata needed for formatting is available before submission in +`/app/input_manifest.json`. Judge feedback reports scores and MP-HPWL metrics; +it should not be needed to discover vector dimensions, macro count, grid bounds, +or MGO decode conventions. The evaluator source stays hidden in the judge +image. -The objective is to minimize MP-HPWL for `superblue1`: +Tiny example +------------ -`raw_score = 100 * (baseline_hpwl - candidate_hpwl) / baseline_hpwl` +Suppose a very small toy instance had this public metadata: -`score = max(0, raw_score)` +```text +node_cnt = 2 +dim = 4 +n_grid_x = 10 +n_grid_y = 8 +``` + +Then this JSON: + +```json +{"placement": [2.0, 7.5, 1.0, 6.0]} +``` + +means: + +```text +macro 0 desired MGO grid coordinate: (2.0, 1.0) +macro 1 desired MGO grid coordinate: (7.5, 6.0) +``` + +The equivalent split x/y form is: + +```json +{"x": [2.0, 7.5], "y": [1.0, 6.0]} +``` + +Minimal valid artifact +---------------------- + +The repository reference solution is: + +```json +{"fill": 0.0} +``` + +A small Python generator that writes a random in-bounds candidate from the +manifest can look like: + +```python +import json +import numpy as np +from pathlib import Path + +manifest = json.loads(Path("/app/input_manifest.json").read_text()) +node_cnt = int(manifest["node_cnt"]) +n_grid_x = float(manifest["n_grid_x"]) +n_grid_y = float(manifest["n_grid_y"]) + +rng = np.random.default_rng(1) +x = rng.uniform(0.0, n_grid_x, size=node_cnt).tolist() +y = rng.uniform(0.0, n_grid_y, size=node_cnt).tolist() + +with open("/app/solution.json", "w", encoding="utf-8") as f: + json.dump({"x": x, "y": y}, f) +``` + +Validity +-------- + +A submission is valid if: + +1. `/app/solution.json` is valid JSON. +2. It describes exactly one placement through `placement`, `x` plus `y`, + `candidate`, `candidates`, or `fill`. +3. The expanded placement has exactly `1024` finite numeric values. +4. X coordinates are in `[0, 224]` and y coordinates are in `[0, 224]`. +5. BBOPlace can legalize the submitted candidate. + +Invalid submissions, out-of-bounds coordinates, malformed JSON, and candidates +that cannot be legalized receive score `0`. + +Scoring +------- + +The objective is to minimize MP-HPWL for `superblue1`: + +```text +raw_score = 100 * (baseline_hpwl - candidate_hpwl) / baseline_hpwl +score = max(0, raw_score) +``` The baseline constant is from the BBOPlace-Bench report, Table V, `MGO + PSO` -MP-HPWL. The paper reports values in units of `x10^5`; the judge stores the -raw HPWL value relaxed by `1.2x`: +MP-HPWL. The report uses units of `x10^5`; the judge stores the raw HPWL value +relaxed by `1.2x`: -- `superblue1`: `0.696e5` +```text +superblue1 0.696e5 +``` Both iterative submissions and final verification evaluate this same single design. The submit helper may still save the best iterative JSON artifact and diff --git a/2.0/problems/bboplace_direct_ispd2005/config.yaml b/2.0/problems/bboplace_direct_ispd2005/config.yaml index 520d27af..ccd80af9 100644 --- a/2.0/problems/bboplace_direct_ispd2005/config.yaml +++ b/2.0/problems/bboplace_direct_ispd2005/config.yaml @@ -2,17 +2,20 @@ tag: optimization runtime: language: json timeout_seconds: 10800 - environment: "JSON placement for one hidden ISPD2005 BBOPlace design" + environment: "JSON placement for one visible ISPD2005 BBOPlace design; hidden evaluator" apt_packages: - python3-numpy docker: image: ubuntu:24.04 judge_image: ghcr.io/frontiercs/frontiercs-bboplace-data:2026-06-ispd-iccad + visible_inputs: + - source: /opt/bboplace-bench/benchmarks/ispd2005/adaptec1 + destination: /app/input/benchmarks/ispd2005/adaptec1 submission: kind: file path: /app/solution.json environment: - cpus: 8 + cpus: 4 memory_mb: 16384 storage_mb: 8192 build_timeout_seconds: 3600 diff --git a/2.0/problems/bboplace_direct_ispd2005/harbor/app/input_manifest.json b/2.0/problems/bboplace_direct_ispd2005/harbor/app/input_manifest.json new file mode 100644 index 00000000..f0c08bb4 --- /dev/null +++ b/2.0/problems/bboplace_direct_ispd2005/harbor/app/input_manifest.json @@ -0,0 +1,88 @@ +{ + "dataset": "ispd2005", + "benchmark": "adaptec1", + "benchmark_path": "/app/input/benchmarks/ispd2005/adaptec1", + "placer": "mgo", + "metric": "mp_hpwl", + "objective": "minimize", + "dim": 1086, + "node_cnt": 543, + "net_cnt": 693, + "canvas_width": 10692.0, + "canvas_height": 10680.0, + "canvas_lx": 459, + "canvas_ly": 459, + "n_grid_x": 224, + "n_grid_y": 224, + "bounds_kind": "mgo_repeated_grid", + "max_candidates_per_submission": 1, + "baseline_hpwl": 696000.0, + "files": [ + "adaptec1.aux", + "adaptec1.dp.aux", + "adaptec1.eplace-ip.pl", + "adaptec1.eplace.aux", + "adaptec1.lg.pl", + "adaptec1.nets", + "adaptec1.nodes", + "adaptec1.pl", + "adaptec1.scl", + "adaptec1.wts" + ], + "submission_path": "/app/solution.json", + "coordinate_format": "placement[0:node_cnt] are x-grid coordinates; placement[node_cnt:2*node_cnt] are y-grid coordinates", + "macro_selection": { + "source_file": "adaptec1.nodes", + "selected_nodes": "terminal nodes", + "count": 543, + "order": "descending width*height area; ties keep adaptec1.nodes file order" + }, + "net_selection": { + "source_file": "adaptec1.nets", + "count": 693, + "rules": [ + "scan NetDegree records and their pin rows", + "keep only pin rows whose node name is one of the 543 selected macros", + "within one net, keep only the first pin row for a repeated macro name", + "discard nets with fewer than two distinct selected macros", + "preserve the remaining net order from adaptec1.nets" + ], + "degree_histogram": { + "2": 516, + "3": 51, + "4": 90, + "5": 13, + "6": 8, + "7": 3, + "8": 10, + "10": 1, + "349": 1 + } + }, + "mgo_decode": { + "grid": [224, 224], + "grid_cell_size": { + "width": 47.732142857142854, + "height": 47.67857142857143 + }, + "bookshelf_to_grid_hint": "Visible .pl coordinates are Bookshelf absolute coordinates. To convert a .pl macro bottom-left coordinate to a requested MGO grid coordinate, use x_grid=(pl_x-canvas_lx)/grid_cell_width and y_grid=(pl_y-canvas_ly)/grid_cell_height.", + "coordinate_bounds": { + "x": [0, 224], + "y": [0, 224] + }, + "coordinate_semantics": "desired MGO grid coordinates; evaluator floors floating coordinates before grid lookup and may move macros during legality repair", + "legalizer_rank": { + "rank_key": "area_sum", + "definition": "For each selected net, net area is the sum of width*height over macros on that net. For each macro, area_sum is the sum of net areas over selected nets containing that macro. Macros are greedily legalized in descending area_sum order; ties keep macro order." + }, + "decode_summary": [ + "candidate[0:543] and candidate[543:1086] are floored to requested grid ids", + "scaled macro size is ceil(width/grid_cell_width), ceil(height/grid_cell_height)", + "for each macro in legalizer_rank order, all grid cells that would overlap already placed macros or exceed the legal grid are masked out", + "among legal cells, the evaluator minimizes incremental HPWL to nets that already have placed macros, using pin offsets from adaptec1.nets", + "ties in incremental HPWL are broken by the L1 physical distance from the requested floored grid coordinate", + "the final macro bottom-left position is grid_id_x*grid_cell_width, grid_id_y*grid_cell_height in the evaluator's canvas-local coordinate system" + ], + "hpwl_summary": "After legalization, MP-HPWL is the sum over the 693 selected nets of (max pin x - min pin x) + (max pin y - min pin y), where pin x/y is macro bottom-left plus half macro size plus the pin offset from adaptec1.nets." + } +} diff --git a/2.0/problems/bboplace_direct_ispd2005/readme b/2.0/problems/bboplace_direct_ispd2005/readme index 8d0a375e..4e160041 100644 --- a/2.0/problems/bboplace_direct_ispd2005/readme +++ b/2.0/problems/bboplace_direct_ispd2005/readme @@ -1,9 +1,73 @@ BBOPlace Direct ISPD2005 ======================= -Directly submit one JSON macro placement for a single BBOPlace ISPD2005 design: -`adaptec1`. The hidden judge evaluates the placement with the original -BBOPlace-Bench MGO MP-HPWL evaluator. +Problem +------- + +Directly submit one JSON macro-placement vector for one visible ISPD2005 +design: `adaptec1`. The hidden judge evaluates your vector with the original +BBOPlace-Bench Mask-Guided Optimization (MGO) MP-HPWL evaluator. + +This is a VLSI chip-placement optimization task. A chip design contains many +rectangular objects. Large objects are called macros, smaller logic objects are +called standard cells, and nets describe which pins on those objects must be +connected by wires. A placement assigns physical locations to movable objects +on a rectangular chip canvas. Good placements keep connected objects near each +other while avoiding illegal macro overlap and boundary violations. + +This direct variant gives you the raw `adaptec1` Bookshelf benchmark files in +the agent workspace. You may parse those files, run local search, generate many +trial JSON files, and call the submit helper for feedback. Only the final +`/app/solution.json` artifact is graded. + +Your JSON vector is not a final Bookshelf `.pl` solution. It is a BBOPlace MGO +genotype: one desired grid coordinate for each macro. The hidden evaluator +decodes the vector by placing macros on a 2D grid, masking out illegal grid +positions, greedily adjusting macros to legal positions, and then reporting +macro-placement half-perimeter wirelength (MP-HPWL). Lower MP-HPWL is better. + +Background definitions +---------------------- + +A net connects two or more pins. The half-perimeter wirelength of one net is: + +```text +(max pin x - min pin x) + (max pin y - min pin y) +``` + +HPWL is the sum of that value over nets. MP-HPWL is the same idea restricted to +the macro-placement evaluation path used by BBOPlace-Bench. The judge reports +only MP-HPWL for scoring; it does not run global placement of standard cells. + +ISPD2005 uses the Bookshelf placement format. For this task the visible files +have these roles: + +```text +.aux lists the component files for this benchmark instance +.nodes object names, sizes, and terminal/fixed status +.nets net connectivity and optional pin offsets +.pl initial or solution locations and orientations +.scl row/floorplan information +.wts optional net or object weights +``` + +Useful parsing notes: + +- Lines beginning with `#` are comments in typical Bookshelf files. +- `.nodes` contains `NumNodes`, `NumTerminals`, then rows like + `name width height` with optional `terminal`. +- `.nets` contains `NumNets`, `NumPins`, then `NetDegree : k net_name` + followed by `k` pin rows. Pin rows name an object and may include x/y pin + offsets after `:`. +- `.pl` contains object locations. Fixed terminals are usually marked with + `/FIXED` or `/FIXED_NI`. +- `.scl` describes placement rows and can be used to infer the legal canvas + region. The official MGO grid bounds for this task are also provided in + `/app/input_manifest.json`. + +You do not need to reproduce the official HPWL or legalizer exactly to submit a +valid JSON. Local parsers are useful only for choosing better coordinates before +calling `/app/submit.sh`. Runtime and resources --------------------- @@ -11,61 +75,199 @@ Runtime and resources Your final submission is a JSON file at `/app/solution.json`. You are encouraged to write Python programs, shell scripts, search loops, local parsers, or any other helper code in `/app` while working. Those programs may generate and -overwrite `/app/solution.json` many times. Only the final JSON artifact is -graded. +overwrite `/app/solution.json` many times. The agent container provides: -- 8 CPU cores +- 4 CPU cores - 16 GiB memory - 8 GiB storage - no GPU - Python 3 with NumPy available for local helper scripts -- hidden benchmark data is not available in the agent workspace +- the `adaptec1` benchmark input is available in the agent workspace at + `/app/input/benchmarks/ispd2005/adaptec1` The judge also runs on CPU only. Do not rely on CUDA, DREAMPlace, Ray, or GPU placement libraries for scoring. +Visible input files +------------------- + +The agent can inspect the raw ISPD2005 placement input while generating +`/app/solution.json`: + +```text +/app/input_manifest.json +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.aux +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.dp.aux +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.eplace-ip.pl +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.eplace.aux +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.lg.pl +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.nodes +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.nets +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.pl +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.scl +/app/input/benchmarks/ispd2005/adaptec1/adaptec1.wts +``` + +The evaluator implementation and scoring runtime remain hidden in the judge +image. + +`/app/input_manifest.json` is the machine-readable public data channel for +instance-specific values and evaluator-facing conventions. Read it instead of +hard-coding dimensions or inferring hidden details from feedback. It includes +the placement dimension, macro count, grid bounds, canvas information, scoring +baseline, macro-selection rule, selected-net rule, and a concise description of +the MGO decode/legalization behavior. Together with the visible Bookshelf files, +that manifest is intended to be sufficient for building a useful local model; +external repositories or internet search should not be needed. + Submission format ----------------- Submit exactly one placement for `adaptec1` by writing `/app/solution.json`. -After your program writes the file, call `bash /app/submit.sh` to score that -JSON. The JSON must use one of these forms: +After your program writes the file, call: + +```bash +bash /app/submit.sh +``` + +to score that JSON through the black-box judge. + +The JSON must use one of these forms: ```json -{"placement": [0.0, 0.0, "..."]} +{"placement": [0.0, 0.0, 1.0, 1.0]} ``` or: ```json -{"x": [0.0, 0.0, "..."], "y": [0.0, 0.0, "..."]} +{"x": [0.0, 1.0], "y": [0.0, 1.0]} ``` +For smoke testing, this shorthand is also accepted: + +```json +{"fill": 0.0} +``` + +`{"fill": v}` expands inside the evaluator to a full vector where every +coordinate is `v`. It is a convenient way to create a syntactically valid smoke +test submission. + The placement vector length must equal `dim = 2 * node_cnt`. The first `node_cnt` entries are x-grid coordinates and the remaining `node_cnt` entries -are y-grid coordinates. Coordinates must be finite, with x in `[0, n_grid_x]` -and y in `[0, n_grid_y]`. Only one placement is accepted. +are y-grid coordinates: + +```text +placement[0:node_cnt] = x-grid coordinates +placement[node_cnt:2 * node_cnt] = y-grid coordinates +``` -The judge discloses public metadata through the iterative feedback path, -including `dim`, `node_cnt`, `n_grid_x`, `n_grid_y`, and the baseline HPWL. -The netlist and evaluator source stay hidden in the judge image. +Coordinates must be finite. X coordinates must be in `[0, n_grid_x]`; y +coordinates must be in `[0, n_grid_y]`. Coordinates may be integers or floats. +Only one placement is accepted. -Score ------ +The public metadata needed for formatting is available before submission in +`/app/input_manifest.json`. Judge feedback reports scores and MP-HPWL metrics; +it should not be needed to discover vector dimensions or grid bounds. The +evaluator source stays hidden in the judge image. -The objective is to minimize MP-HPWL for `adaptec1`: +Tiny example +------------ -`raw_score = 100 * (baseline_hpwl - candidate_hpwl) / baseline_hpwl` +Suppose a very small toy instance had this public metadata: -`score = max(0, raw_score)` +```text +node_cnt = 2 +dim = 4 +n_grid_x = 10 +n_grid_y = 8 +``` + +Then this JSON: + +```json +{"placement": [2.0, 7.5, 1.0, 6.0]} +``` + +means: + +```text +macro 0 desired MGO grid coordinate: (2.0, 1.0) +macro 1 desired MGO grid coordinate: (7.5, 6.0) +``` + +The equivalent split x/y form is: + +```json +{"x": [2.0, 7.5], "y": [1.0, 6.0]} +``` + +Minimal valid artifact +---------------------- + +The repository reference solution is: + +```json +{"fill": 0.0} +``` + +A small Python generator that writes a random in-bounds candidate from the +manifest can look like: + +```python +import json +import numpy as np +from pathlib import Path + +manifest = json.loads(Path("/app/input_manifest.json").read_text()) +node_cnt = int(manifest["node_cnt"]) +n_grid_x = float(manifest["n_grid_x"]) +n_grid_y = float(manifest["n_grid_y"]) + +rng = np.random.default_rng(1) +x = rng.uniform(0.0, n_grid_x, size=node_cnt).tolist() +y = rng.uniform(0.0, n_grid_y, size=node_cnt).tolist() + +with open("/app/solution.json", "w", encoding="utf-8") as f: + json.dump({"x": x, "y": y}, f) +``` + +Validity +-------- + +A submission is valid if: + +1. `/app/solution.json` is valid JSON. +2. It describes exactly one placement through `placement`, `x` plus `y`, or + `fill`. +3. The expanded placement has exactly `dim` finite numeric values. +4. X coordinates are in `[0, n_grid_x]` and y coordinates are in + `[0, n_grid_y]`. +5. BBOPlace can legalize the submitted candidate. + +Invalid submissions, out-of-bounds coordinates, malformed JSON, and candidates +that cannot be legalized receive score `0`. + +Scoring +------- + +The objective is to minimize MP-HPWL for `adaptec1`: + +```text +raw_score = 100 * (baseline_hpwl - candidate_hpwl) / baseline_hpwl +score = max(0, raw_score) +``` The baseline constant is from the BBOPlace-Bench report, Table III, -`MGO + Vanilla-EA` MP-HPWL. The paper reports values in units of `x10^5`; the -judge stores the raw HPWL value relaxed by `1.2x`: +`MGO + Vanilla-EA` MP-HPWL. The report uses units of `x10^5`; the judge stores +the raw HPWL value relaxed by `1.2x`: -- `adaptec1`: `6.96e5` +```text +adaptec1 6.96e5 +``` Both iterative submissions and final verification evaluate this same single design. The submit helper may still save the best iterative JSON artifact and diff --git a/2.0/problems/bboplace_iccad2015/config.yaml b/2.0/problems/bboplace_iccad2015/config.yaml index cf5bc374..423cb385 100644 --- a/2.0/problems/bboplace_iccad2015/config.yaml +++ b/2.0/problems/bboplace_iccad2015/config.yaml @@ -9,7 +9,7 @@ runtime: image: ubuntu:24.04 judge_image: ghcr.io/frontiercs/frontiercs-bboplace-data:2026-06-ispd-iccad environment: - cpus: 8 + cpus: 4 memory_mb: 16384 storage_mb: 8192 build_timeout_seconds: 3600 diff --git a/2.0/problems/bboplace_iccad2015/readme b/2.0/problems/bboplace_iccad2015/readme index e2e413b4..0292b47f 100644 --- a/2.0/problems/bboplace_iccad2015/readme +++ b/2.0/problems/bboplace_iccad2015/readme @@ -1,66 +1,248 @@ BBOPlace ICCAD2015 ================== -Write a Python solution that proposes macro placements for the BBOPlace MGO -formulation on the ICCAD2015 benchmark suite. The hidden judge evaluates your -placements with the original BBOPlace-Bench MP-HPWL evaluator. +Problem +------- + +Write a Python solution that proposes macro placements for the BBOPlace +Mask-Guided Optimization (MGO) formulation on eight ICCAD2015 placement +benchmarks: + +```text +superblue1, superblue3, superblue4, superblue5, +superblue7, superblue10, superblue16, superblue18 +``` + +This is a VLSI chip-placement optimization task. A chip design contains many +rectangular objects. Large objects are called macros, smaller logic objects are +called standard cells, and nets describe which pins on those objects must be +connected by wires. A placement assigns physical locations to the movable +objects on a rectangular chip canvas. Good placements keep connected objects +near each other while avoiding illegal macro overlap and boundary violations. + +This task uses the BBOPlace-Bench MGO representation. Your submitted vector is +not a final physical DEF placement. It is a genotype: one desired grid +coordinate for each macro. The hidden BBOPlace evaluator decodes that genotype +by placing macros on a 2D grid, masking out illegal grid positions, greedily +adjusting macros to legal positions, and then reporting macro-placement +half-perimeter wirelength (MP-HPWL). Lower MP-HPWL is better. + +You do not need to implement an ICCAD2015 parser, a legalizer, HPWL +computation, or BBOPlace itself for this suite task. The judge owns the hidden +benchmark files and the original BBOPlace-Bench evaluator. Your job is to +generate promising candidate coordinate vectors from the public metadata passed +to `solve(info)`. External repositories, papers, internet search, and BBOPlace +source code are not needed for this task. Do not try to fetch or reconstruct +the hidden evaluator. Treat it as a black box and use only the task statement, +`AGENT.md`, the `info` dictionary passed to your solution, and feedback +returned by `/app/submit.sh`. + +Background definitions +---------------------- + +A net connects two or more pins. The half-perimeter wirelength of one net is: + +```text +(max pin x - min pin x) + (max pin y - min pin y) +``` + +HPWL is the sum of that value over nets. MP-HPWL is the same idea restricted to +the macro-placement evaluation path used by BBOPlace-Bench. The judge reports +only MP-HPWL for scoring; it does not run global placement of standard cells. + +ICCAD2015 contest designs are distributed in an LEF/DEF-style flow. The +important file roles are: + +```text +.def design components, pins, nets, floorplan, and placement data +.lef technology and macro geometry +.v logical netlist +.sdc timing constraints +*_Early.lib early timing library +*_Late.lib late timing library +``` + +Those files are hidden for this suite task. They are described here only so the +optimization problem is clear. Runtime and resources --------------------- Your solution runs in the main agent container with: -- 8 CPU cores +- 4 CPU cores - 16 GiB memory - 8 GiB storage - no GPU - Python 3 with NumPy available -- internet access may be available during the trial, but the benchmark data is - hidden and is not available in the agent workspace The final verifier timeout is 10800 seconds. The judge also runs on CPU only. Do not rely on CUDA, DREAMPlace, Ray, or GPU placement libraries for scoring; the official metric path used here is MGO with MP-HPWL. -Your file must define one of: +Submission interface +-------------------- + +Create `/app/solution.py`. It must define one of: - `solve(info)` - `generate(info)` - `run(info)` - `CANDIDATES`, `CANDIDATE`, or `PLACEMENT` -The recommended interface is `solve(info)`. The judge calls it once per -benchmark. `info` contains: +The recommended interface is `solve(info)`. The judge imports the file in an +isolated process and calls it once per selected benchmark. `info` is a +JSON-like dictionary containing: -- `benchmark`: one of `superblue1`, `superblue3`, `superblue4`, `superblue5`, - `superblue7`, `superblue10`, `superblue16`, `superblue18` -- `dim`: placement vector length -- `node_cnt`: number of macros -- `n_grid_x`, `n_grid_y`: MGO grid bounds -- `max_candidates_per_submission`: 16 -- `baseline_hpwl`: the baseline HPWL used for scoring +```text +dataset: "iccad2015" +benchmark: one of superblue1, superblue3, superblue4, superblue5, + superblue7, superblue10, superblue16, superblue18 +placer: "mgo" +metric: "mp_hpwl" +objective: "minimize" +dim: placement vector length, always 2 * node_cnt +node_cnt: number of movable macros represented by the vector +net_cnt: number of nets in the benchmark +canvas_width, canvas_height: physical canvas dimensions +n_grid_x, n_grid_y: inclusive MGO grid-coordinate upper bounds +bounds_kind: "mgo_repeated_grid" +max_candidates_per_submission: 16 +baseline_hpwl: the HPWL baseline used for scoring +``` -Return either one placement vector of length `dim`, or a 2D list/array with up -to 16 placement candidates. For MGO, the first `node_cnt` entries are x-grid -coordinates in `[0, n_grid_x]`, and the remaining `node_cnt` entries are y-grid -coordinates in `[0, n_grid_y]`. +The `info` dictionary is the public data channel for benchmark-specific +constants in this suite task. The raw ICCAD2015 files stay hidden, but the +constants needed to create a syntactically valid placement are provided on +every call to `solve(info)`. In particular, do not assume that the +quick-feedback benchmark dimensions also apply to the full suite. Read `dim`, +`node_cnt`, `n_grid_x`, `n_grid_y`, `canvas_width`, `canvas_height`, and +`baseline_hpwl` from the `info` argument each time the function is called. -Score ------ +Because the raw netlist and macro order are hidden, a useful solver should +return several robust coordinate patterns rather than trying to compute true +HPWL locally. Examples include zero or center fills, edge/corner spreads, +diagonal or striped layouts, clustered layouts, deterministic random layouts, +and small deterministic variants keyed by `info["benchmark"]`. The judge will +evaluate up to 16 candidates and keep the best legal candidate for each +benchmark. -The objective is to minimize MP-HPWL. For each benchmark: +The final verifier calls your solution separately for: -`raw_score = 100 * (baseline_hpwl - candidate_hpwl) / baseline_hpwl` +```text +superblue1, superblue3, superblue4, superblue5, +superblue7, superblue10, superblue16, superblue18 +``` -`score = max(0, raw_score)` +Each call has the same field names but may have different values. A correct +solution can therefore be written without knowing or hard-coding the per-design +constants in advance: -The final score is the mean score over the eight ICCAD2015 benchmarks. The -unbounded score is the mean raw score before the zero clip. +```python +def solve(info): + n = int(info["node_cnt"]) + gx = float(info["n_grid_x"]) + gy = float(info["n_grid_y"]) + x = [0.5 * gx] * n + y = [0.5 * gy] * n + return x + y +``` + +Return either one placement vector of length `dim`, or a 2D list/NumPy array +with up to 16 placement candidates. For every candidate: + +```text +candidate[0:node_cnt] = x-grid coordinates +candidate[node_cnt:2 * node_cnt] = y-grid coordinates +x coordinates must be finite and in [0, n_grid_x] +y coordinates must be finite and in [0, n_grid_y] +``` + +Coordinates may be integers or floats. They are interpreted by BBOPlace's MGO +decoder as desired grid positions; the decoder may move macros during legality +repair. If you return multiple candidates, the judge evaluates all of them and +uses the candidate with the lowest legal MP-HPWL for that benchmark. + +A minimal valid solution is: + +```python +def solve(info): + return [0.0] * int(info["dim"]) +``` + +Tiny example +------------ + +Suppose the judge called `solve(info)` with this small illustrative metadata: + +```python +info = { + "benchmark": "toy", + "node_cnt": 2, + "dim": 4, + "n_grid_x": 10, + "n_grid_y": 8, +} +``` + +Then a valid single candidate could be: + +```python +[2.0, 7.5, 1.0, 6.0] +``` + +The first `node_cnt = 2` values are x-grid coordinates, and the last two values +are y-grid coordinates. This candidate asks MGO to place macro 0 near grid +`(2.0, 1.0)` and macro 1 near grid `(7.5, 6.0)`. A two-candidate return value +for the same toy metadata could be: + +```python +[ + [2.0, 7.5, 1.0, 6.0], + [0.0, 10.0, 0.0, 8.0], +] +``` + +A simple multi-candidate skeleton is: + +```python +import numpy as np + + +def solve(info): + n = int(info["node_cnt"]) + dim = int(info["dim"]) + gx = float(info["n_grid_x"]) + gy = float(info["n_grid_y"]) + name = str(info["benchmark"]) + seed = sum((i + 1) * ord(ch) for i, ch in enumerate(name)) % (2**32) + rng = np.random.default_rng(seed) + + candidates = [] + candidates.append(np.zeros(dim)) + for _ in range(15): + x = rng.uniform(0.0, gx, size=n) + y = rng.uniform(0.0, gy, size=n) + candidates.append(np.concatenate([x, y])) + return np.asarray(candidates) +``` + +Local harness +------------- + +The official evaluator uses hidden data and a black-box judge. During a Harbor +trial you may call: + +```bash +bash /app/submit.sh +``` + +to submit the current `/app/solution.py` and receive public score feedback. During iterative agent submissions, `/app/submit.sh` gives quick feedback on -`superblue1` only. The final verifier evaluates the full eight-benchmark suite. -Use the quick feedback to debug general placement logic, not as the complete -leaderboard score. +`superblue1` only. The final verifier evaluates the full eight-benchmark +suite. Use the quick feedback to debug general placement logic, candidate +formatting, and search ideas, not as the complete leaderboard score. The submit helper saves the best quick-feedback artifact it has seen. During final verification, the verifier reruns both the current `/app/solution.py` and @@ -68,18 +250,46 @@ that saved best iterative artifact on the full suite, then uses the better full-suite score. A quick-feedback score is never used directly as the final reward. +Validity +-------- + +A submission is valid if: + +1. `/app/solution.py` imports successfully with Python 3. +2. It exposes one of the accepted interfaces listed above. +3. For each benchmark call, it returns one to 16 candidates. +4. Every candidate has exactly `dim` finite numeric values. +5. X coordinates are in `[0, n_grid_x]` and y coordinates are in + `[0, n_grid_y]`. +6. BBOPlace can legalize at least one candidate for each evaluated benchmark. + +Invalid submissions, timeouts, and candidates that cannot be legalized receive +score `0`. + +Scoring +------- + +The objective is to minimize MP-HPWL. For each benchmark: + +```text +raw_score = 100 * (baseline_hpwl - candidate_hpwl) / baseline_hpwl +score = max(0, raw_score) +``` + +The final score is the mean score over the eight ICCAD2015 benchmarks. The +unbounded score is the mean raw score before the zero clip. + The baseline constants are from the BBOPlace-Bench report, Table V, `MGO + PSO` MP-HPWL. The paper reports values in units of `x10^5`; the judge stores raw HPWL values relaxed by `1.2x`: -- `superblue1`: `0.696e5` -- `superblue3`: `1.824e5` -- `superblue4`: `1.128e5` -- `superblue5`: `4.512e5` -- `superblue7`: `2.028e5` -- `superblue10`: `0.648e5` -- `superblue16`: `1.152e5` -- `superblue18`: `0.576e5` - -The benchmark data and evaluator source are only present in the judge image. -They are not available in the agent workspace. +```text +superblue1: 0.696e5 +superblue3: 1.824e5 +superblue4: 1.128e5 +superblue5: 4.512e5 +superblue7: 2.028e5 +superblue10: 0.648e5 +superblue16: 1.152e5 +superblue18: 0.576e5 +``` diff --git a/2.0/problems/bboplace_ispd2005/config.yaml b/2.0/problems/bboplace_ispd2005/config.yaml index 79d071b2..35f391a2 100644 --- a/2.0/problems/bboplace_ispd2005/config.yaml +++ b/2.0/problems/bboplace_ispd2005/config.yaml @@ -5,11 +5,13 @@ runtime: environment: "Python solution returning BBOPlace MGO placement candidates; hidden ISPD2005 judge data" apt_packages: - python3-numpy + - python3-matplotlib + - python3-pil docker: image: ubuntu:24.04 judge_image: ghcr.io/frontiercs/frontiercs-bboplace-data:2026-06-ispd-iccad environment: - cpus: 8 + cpus: 4 memory_mb: 16384 storage_mb: 8192 build_timeout_seconds: 3600 diff --git a/2.0/problems/bboplace_ispd2005/readme b/2.0/problems/bboplace_ispd2005/readme index a022ec2b..d15c4d47 100644 --- a/2.0/problems/bboplace_ispd2005/readme +++ b/2.0/problems/bboplace_ispd2005/readme @@ -1,66 +1,243 @@ BBOPlace ISPD2005 ================= -Write a Python solution that proposes macro placements for the BBOPlace MGO -formulation on the ISPD2005 benchmark suite. The hidden judge evaluates your -placements with the original BBOPlace-Bench MP-HPWL evaluator. +Problem +------- + +Write a Python solution that proposes macro placements for the BBOPlace +Mask-Guided Optimization (MGO) formulation on six ISPD2005 placement +benchmarks: + +```text +adaptec1, adaptec2, adaptec3, adaptec4, bigblue1, bigblue3 +``` + +This is a VLSI chip-placement optimization task. A chip design contains many +rectangular objects. Large objects are called macros, smaller logic objects are +called standard cells, and nets describe which pins on those objects must be +connected by wires. A placement assigns physical locations to the movable +objects on a rectangular chip canvas. Good placements keep connected objects +near each other while avoiding illegal macro overlap and boundary violations. + +This task uses the BBOPlace-Bench MGO representation. Your submitted vector is +not a final physical `.pl` file. It is a genotype: one desired grid coordinate +for each macro. The hidden BBOPlace evaluator decodes that genotype by placing +macros on a 2D grid, masking out illegal grid positions, greedily adjusting +macros to legal positions, and then reporting macro-placement half-perimeter +wirelength (MP-HPWL). Lower MP-HPWL is better. + +You do not need to implement an ISPD parser, a legalizer, HPWL computation, or +BBOPlace itself for this suite task. The judge owns the hidden benchmark files +and the original BBOPlace-Bench evaluator. Your job is to generate promising +candidate coordinate vectors from the public metadata passed to `solve(info)`. +External repositories, papers, internet search, and BBOPlace source code are +not needed for this task. Do not try to fetch or reconstruct the hidden +evaluator. Treat it as a black box and use only the task statement, `AGENT.md`, +the `info` dictionary passed to your solution, and feedback returned by +`/app/submit.sh`. + +Background definitions +---------------------- + +A net connects two or more pins. The half-perimeter wirelength of one net is: + +```text +(max pin x - min pin x) + (max pin y - min pin y) +``` + +HPWL is the sum of that value over nets. MP-HPWL is the same idea restricted to +the macro-placement evaluation path used by BBOPlace-Bench. The judge reports +only MP-HPWL for scoring; it does not run global placement of standard cells. + +ISPD2005 designs are stored in the Bookshelf placement format. The important +file roles are: + +```text +.aux lists the component files for one benchmark instance +.nodes object names, sizes, and terminal/fixed status +.nets net connectivity and optional pin offsets +.pl initial or solution locations and orientations +.scl row/floorplan information +.wts optional net or object weights +``` + +Those files are hidden for this suite task. They are described here only so the +optimization problem is clear. Runtime and resources --------------------- Your solution runs in the main agent container with: -- 8 CPU cores +- 4 CPU cores - 16 GiB memory - 8 GiB storage - no GPU - Python 3 with NumPy available -- internet access may be available during the trial, but the benchmark data is - hidden and is not available in the agent workspace The final verifier timeout is 10800 seconds. The judge also runs on CPU only. Do not rely on CUDA, DREAMPlace, Ray, or GPU placement libraries for scoring; the official metric path used here is MGO with MP-HPWL. -Your file must define one of: +Submission interface +-------------------- + +Create `/app/solution.py`. It must define one of: - `solve(info)` - `generate(info)` - `run(info)` - `CANDIDATES`, `CANDIDATE`, or `PLACEMENT` -The recommended interface is `solve(info)`. The judge calls it once per -benchmark. `info` contains: +The recommended interface is `solve(info)`. The judge imports the file in an +isolated process and calls it once per selected benchmark. `info` is a JSON-like +dictionary containing: -- `benchmark`: one of `adaptec1`, `adaptec2`, `adaptec3`, `adaptec4`, - `bigblue1`, `bigblue3` -- `dim`: placement vector length -- `node_cnt`: number of macros -- `n_grid_x`, `n_grid_y`: MGO grid bounds -- `max_candidates_per_submission`: 16 -- `baseline_hpwl`: the baseline HPWL used for scoring +```text +dataset: "ispd2005" +benchmark: one of adaptec1, adaptec2, adaptec3, adaptec4, bigblue1, bigblue3 +placer: "mgo" +metric: "mp_hpwl" +objective: "minimize" +dim: placement vector length, always 2 * node_cnt +node_cnt: number of movable macros represented by the vector +net_cnt: number of nets in the benchmark +canvas_width, canvas_height: physical canvas dimensions +n_grid_x, n_grid_y: inclusive MGO grid-coordinate upper bounds +bounds_kind: "mgo_repeated_grid" +max_candidates_per_submission: 16 +baseline_hpwl: the HPWL baseline used for scoring +``` -Return either one placement vector of length `dim`, or a 2D list/array with up -to 16 placement candidates. For MGO, the first `node_cnt` entries are x-grid -coordinates in `[0, n_grid_x]`, and the remaining `node_cnt` entries are y-grid -coordinates in `[0, n_grid_y]`. +The `info` dictionary is the public data channel for benchmark-specific +constants in this suite task. The raw ISPD2005 Bookshelf files stay hidden, but +the constants needed to create a syntactically valid placement are provided on +every call to `solve(info)`. In particular, do not assume that the quick-feedback +benchmark dimensions also apply to the full suite. Read `dim`, `node_cnt`, +`n_grid_x`, `n_grid_y`, `canvas_width`, `canvas_height`, and `baseline_hpwl` +from the `info` argument each time the function is called. -Score ------ +Because the raw netlist and macro order are hidden, a useful solver should +return several robust coordinate patterns rather than trying to compute true +HPWL locally. Examples include zero or center fills, edge/corner spreads, +diagonal or striped layouts, clustered layouts, deterministic random layouts, +and small deterministic variants keyed by `info["benchmark"]`. The judge will +evaluate up to 16 candidates and keep the best legal candidate for each +benchmark. -The objective is to minimize MP-HPWL. For each benchmark: +The final verifier calls your solution separately for: -`raw_score = 100 * (baseline_hpwl - candidate_hpwl) / baseline_hpwl` +```text +adaptec1, adaptec2, adaptec3, adaptec4, bigblue1, bigblue3 +``` -`score = max(0, raw_score)` +Each call has the same field names but may have different values. A correct +solution can therefore be written without knowing or hard-coding the per-design +constants in advance: -The final score is the mean score over the six ISPD2005 benchmarks. The -unbounded score is the mean raw score before the zero clip. +```python +def solve(info): + n = int(info["node_cnt"]) + gx = float(info["n_grid_x"]) + gy = float(info["n_grid_y"]) + x = [0.5 * gx] * n + y = [0.5 * gy] * n + return x + y +``` + +Return either one placement vector of length `dim`, or a 2D list/NumPy array +with up to 16 placement candidates. For every candidate: + +```text +candidate[0:node_cnt] = x-grid coordinates +candidate[node_cnt:2 * node_cnt] = y-grid coordinates +x coordinates must be finite and in [0, n_grid_x] +y coordinates must be finite and in [0, n_grid_y] +``` + +Coordinates may be integers or floats. They are interpreted by BBOPlace's MGO +decoder as desired grid positions; the decoder may move macros during legality +repair. If you return multiple candidates, the judge evaluates all of them and +uses the candidate with the lowest legal MP-HPWL for that benchmark. + +A minimal valid solution is: + +```python +def solve(info): + return [0.0] * int(info["dim"]) +``` + +Tiny example +------------ + +Suppose the judge called `solve(info)` with this small illustrative metadata: + +```python +info = { + "benchmark": "toy", + "node_cnt": 2, + "dim": 4, + "n_grid_x": 10, + "n_grid_y": 8, +} +``` + +Then a valid single candidate could be: + +```python +[2.0, 7.5, 1.0, 6.0] +``` + +The first `node_cnt = 2` values are x-grid coordinates, and the last two values +are y-grid coordinates. This candidate asks MGO to place macro 0 near grid +`(2.0, 1.0)` and macro 1 near grid `(7.5, 6.0)`. A two-candidate return value +for the same toy metadata could be: + +```python +[ + [2.0, 7.5, 1.0, 6.0], + [0.0, 10.0, 0.0, 8.0], +] +``` + +A simple multi-candidate skeleton is: + +```python +import numpy as np + + +def solve(info): + n = int(info["node_cnt"]) + dim = int(info["dim"]) + gx = float(info["n_grid_x"]) + gy = float(info["n_grid_y"]) + rng = np.random.default_rng(abs(hash(info["benchmark"])) % (2**32)) + + candidates = [] + candidates.append(np.zeros(dim)) + for _ in range(15): + x = rng.uniform(0.0, gx, size=n) + y = rng.uniform(0.0, gy, size=n) + candidates.append(np.concatenate([x, y])) + return np.asarray(candidates) +``` + +Local harness +------------- + +The official evaluator uses hidden data and a black-box judge. During a Harbor +trial you may call: + +```bash +bash /app/submit.sh +``` + +to submit the current `/app/solution.py` and receive public score feedback. During iterative agent submissions, `/app/submit.sh` gives quick feedback on `adaptec1` only. The final verifier evaluates the full six-benchmark suite. -Use the quick feedback to debug general placement logic, not as the complete -leaderboard score. +Use the quick feedback to debug general placement logic, candidate formatting, +and search ideas, not as the complete leaderboard score. The submit helper saves the best quick-feedback artifact it has seen. During final verification, the verifier reruns both the current `/app/solution.py` and @@ -68,16 +245,31 @@ that saved best iterative artifact on the full suite, then uses the better full-suite score. A quick-feedback score is never used directly as the final reward. -The baseline constants are from the BBOPlace-Bench report, Table III, -`MGO + Vanilla-EA` MP-HPWL. The paper reports values in units of `x10^5`; the -judge stores raw HPWL values relaxed by `1.2x`: +Validity +-------- + +A submission is valid if: + +1. `/app/solution.py` imports successfully with Python 3. +2. It exposes one of the accepted interfaces listed above. +3. For each benchmark call, it returns one to 16 candidates. +4. Every candidate has exactly `dim` finite numeric values. +5. X coordinates are in `[0, n_grid_x]` and y coordinates are in + `[0, n_grid_y]`. +6. BBOPlace can legalize at least one candidate for each evaluated benchmark. + +Invalid submissions, timeouts, and candidates that cannot be legalized receive +score `0`. -- `adaptec1`: `6.96e5` -- `adaptec2`: `73.752e5` -- `adaptec3`: `67.356e5` -- `adaptec4`: `68.148e5` -- `bigblue1`: `2.76e5` -- `bigblue3`: `62.88e5` +Scoring +------- -The benchmark data and evaluator source are only present in the judge image. -They are not available in the agent workspace. +The objective is to minimize MP-HPWL. For each benchmark: + +```text +raw_score = 100 * (baseline_hpwl - candidate_hpwl) / baseline_hpwl +score = max(0, raw_score) +``` + +The final score is the mean score over the six ISPD2005 benchmarks. The +unbounded score is the mean raw score before the zero clip. diff --git a/adapters/frontier-cs-2.0/src/frontier_cs_2_0/adapter.py b/adapters/frontier-cs-2.0/src/frontier_cs_2_0/adapter.py index 8e93cc93..af2e05ae 100644 --- a/adapters/frontier-cs-2.0/src/frontier_cs_2_0/adapter.py +++ b/adapters/frontier-cs-2.0/src/frontier_cs_2_0/adapter.py @@ -217,6 +217,12 @@ def _write_environment( "{extra_apt_install}", extra_apt_install ).replace( "{extra_pip_install}", extra_pip_install + ).replace( + "{visible_input_stages}", + self._visible_input_stages(problem, default_image=judge_image), + ).replace( + "{visible_input_copies}", + self._visible_input_copies(problem), ), encoding="utf-8", ) @@ -234,7 +240,14 @@ def _write_environment( generated_harbor_app_dir.mkdir(parents=True, exist_ok=True) if harbor_app_dir.exists(): shutil.copytree( - harbor_app_dir, generated_harbor_app_dir, dirs_exist_ok=True + harbor_app_dir, + generated_harbor_app_dir, + dirs_exist_ok=True, + ignore=( + shutil.ignore_patterns("input") + if self._visible_inputs(problem) + else None + ), ) judge_dockerfile = ( @@ -300,6 +313,55 @@ def _write_submission_config(self, env_dir: Path, problem: FrontierCS20Problem) json.dumps(submission, indent=2), encoding="utf-8" ) + def _visible_inputs(self, problem: FrontierCS20Problem) -> list[dict[str, str]]: + runtime = problem.config.get("runtime", {}) or {} + entries = runtime.get("visible_inputs", []) or [] + if not isinstance(entries, list): + raise TypeError(f"{problem.problem_id}: runtime.visible_inputs must be a list") + normalized: list[dict[str, str]] = [] + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise TypeError( + f"{problem.problem_id}: runtime.visible_inputs[{index}] must be a mapping" + ) + source = str(entry.get("source") or entry.get("from_image_path") or "") + destination = str(entry.get("destination") or entry.get("to_app_path") or "") + image = str(entry.get("image") or entry.get("from_image") or "") + if not source.startswith("/") or not destination.startswith("/"): + raise ValueError( + f"{problem.problem_id}: visible input paths must be absolute" + ) + for value in (source, destination, image): + if value and any(ch.isspace() for ch in value): + raise ValueError( + f"{problem.problem_id}: visible input values may not contain whitespace" + ) + normalized.append( + {"source": source, "destination": destination, "image": image} + ) + return normalized + + def _visible_input_stages( + self, problem: FrontierCS20Problem, *, default_image: str + ) -> str: + lines: list[str] = [] + for index, entry in enumerate(self._visible_inputs(problem)): + image = entry["image"] or default_image + lines.append(f"FROM {image} AS visible_input_{index}\n") + return "".join(lines) + + def _visible_input_copies(self, problem: FrontierCS20Problem) -> str: + lines: list[str] = [] + for index, entry in enumerate(self._visible_inputs(problem)): + lines.append( + "COPY --from=visible_input_{index} {source} {destination}\n".format( + index=index, + source=entry["source"], + destination=entry["destination"], + ) + ) + return "".join(lines) + def _write_tests( self, task_paths: "TaskPaths", diff --git a/adapters/frontier-cs-2.0/src/frontier_cs_2_0/task-template/environment/Dockerfile b/adapters/frontier-cs-2.0/src/frontier_cs_2_0/task-template/environment/Dockerfile index ef0b014b..8f3f6233 100644 --- a/adapters/frontier-cs-2.0/src/frontier_cs_2_0/task-template/environment/Dockerfile +++ b/adapters/frontier-cs-2.0/src/frontier_cs_2_0/task-template/environment/Dockerfile @@ -1,4 +1,4 @@ -FROM {base_image} +{visible_input_stages}FROM {base_image} RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -29,4 +29,5 @@ COPY readme config.yaml task_config.json submission_config.json AGENT.md \ submit.py submit.sh submissions.py submissions.sh \ wait_submission.py wait_submission.sh cancel_submission.py cancel_submission.sh /app/ COPY harbor_app/ /app/ +{visible_input_copies} RUN chmod +x /app/submit.sh /app/submissions.sh /app/wait_submission.sh /app/cancel_submission.sh diff --git a/tools/bboplace/Dockerfile.data b/tools/bboplace/Dockerfile.data index ace95e01..05b693e9 100644 --- a/tools/bboplace/Dockerfile.data +++ b/tools/bboplace/Dockerfile.data @@ -29,9 +29,37 @@ root = Path("/opt/bboplace-bench") required = [ root / "src" / "evaluator.py", root / "config" / "benchmark.py", - root / "benchmarks" / "ispd2005" / "adaptec1", - root / "benchmarks" / "iccad2015" / "superblue1", ] +for dataset, names in { + "ispd2005": ( + "adaptec1", + "adaptec2", + "adaptec3", + "adaptec4", + "bigblue1", + "bigblue3", + ), + "iccad2015": ( + "superblue1", + "superblue3", + "superblue4", + "superblue5", + "superblue7", + "superblue10", + "superblue16", + "superblue18", + ), +}.items(): + for name in names: + benchmark_dir = root / "benchmarks" / dataset / name + required.append(benchmark_dir) + required += [ + benchmark_dir / f"{name}.aux", + benchmark_dir / f"{name}.nodes", + benchmark_dir / f"{name}.nets", + benchmark_dir / f"{name}.pl", + benchmark_dir / f"{name}.scl", + ] for path in required: if not path.exists(): raise SystemExit(f"missing required BBOPlace data image path: {path}") diff --git a/tools/bboplace/README.md b/tools/bboplace/README.md index 95515a85..301997d0 100644 --- a/tools/bboplace/README.md +++ b/tools/bboplace/README.md @@ -21,9 +21,9 @@ Expected layout: ``` The data sources, SHA256 checksums, observed sizes, and scoring constants are -tracked in `data_manifest.json`. ISPD2005 archives contain gzipped inner files; -the final image must store the uncompressed `.aux`, `.nodes`, `.nets`, `.pl`, -`.scl`, and `.wts` files because the BBOPlace reader expects those filenames. +tracked in `data_manifest.json`. The final image must store the extracted +`.aux`, `.nodes`, `.nets`, `.pl`, `.scl`, and `.wts` files because the +BBOPlace reader expects those filenames. The Frontier-CS evaluator uses only the BBOPlace MGO + MP-HPWL path. It avoids the original BBOPlace `src.evaluator` import path because that path imports @@ -38,8 +38,19 @@ metadata through `solve(info)`. Local validation commands: ```bash -docker build -t frontiercs-bboplace-data:local /path/to/context -docker tag frontiercs-bboplace-data:local ghcr.io/frontiercs/frontiercs-bboplace-data:2026-06-ispd-iccad +mkdir -p /tmp/bboplace-data/benchmarks +git clone https://github.com/lamda-bbo/BBOPlace-Bench /tmp/bboplace-data/BBOPlace-Bench +git -C /tmp/bboplace-data/BBOPlace-Bench checkout 4a0dde451e40a3f368e501df6f027c442fcca02a + +# Download and extract the BBOPlace-Bench README Google Drive archives: +# ispd2005.zip -> /tmp/bboplace-data/benchmarks/ispd2005 +# iccad2015_benchmark.zip -> /tmp/bboplace-data/benchmarks/iccad2015 + +docker build \ + -t ghcr.io/frontiercs/frontiercs-bboplace-data:2026-06-ispd-iccad \ + -f tools/bboplace/Dockerfile.data \ + /tmp/bboplace-data + python3 tools/bboplace/check_constants.py python3 tools/bboplace/check_generated_tasks.py /path/to/generated/frontier-cs-2.0 ``` diff --git a/tools/bboplace/check_constants.py b/tools/bboplace/check_constants.py index f6d37fef..f5de98bf 100644 --- a/tools/bboplace/check_constants.py +++ b/tools/bboplace/check_constants.py @@ -10,10 +10,18 @@ ROOT = Path(__file__).resolve().parents[2] MANIFEST = ROOT / "tools" / "bboplace" / "data_manifest.json" -EVALUATORS = { - "ispd2005": ROOT / "2.0" / "problems" / "bboplace_ispd2005" / "evaluator.py", - "iccad2015": ROOT / "2.0" / "problems" / "bboplace_iccad2015" / "evaluator.py", -} +EVALUATORS = ( + ( + "ispd2005", + ROOT / "2.0" / "problems" / "bboplace_ispd2005" / "evaluator.py", + None, + ), + ( + "iccad2015", + ROOT / "2.0" / "problems" / "bboplace_iccad2015" / "evaluator.py", + None, + ), +) def load_module(path: Path): @@ -27,19 +35,20 @@ def load_module(path: Path): def main() -> int: manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) - for dataset, evaluator_path in EVALUATORS.items(): + for dataset, evaluator_path, benchmark_override in EVALUATORS: module = load_module(evaluator_path) - expected_scored = manifest["datasets"][dataset]["benchmarks_scored"] - expected_constants = manifest["scoring_constants"][dataset]["baseline_hpwl"] + expected_scored = benchmark_override or manifest["datasets"][dataset]["benchmarks_scored"] + all_constants = manifest["scoring_constants"][dataset]["baseline_hpwl"] + expected_constants = {key: all_constants[key] for key in expected_scored} actual_scored = list(module.BENCHMARKS) actual_constants = {key: float(value) for key, value in module.BASELINE_HPWL.items()} if actual_scored != expected_scored: raise SystemExit( - f"{dataset}: benchmark list mismatch: {actual_scored} != {expected_scored}" + f"{evaluator_path}: benchmark list mismatch: {actual_scored} != {expected_scored}" ) if actual_constants != expected_constants: raise SystemExit( - f"{dataset}: baseline constants mismatch: {actual_constants} != {expected_constants}" + f"{evaluator_path}: baseline constants mismatch: {actual_constants} != {expected_constants}" ) print("BBOPlace constants match data_manifest.json") return 0 diff --git a/tools/bboplace/check_generated_tasks.py b/tools/bboplace/check_generated_tasks.py index 80d94ec6..74135437 100644 --- a/tools/bboplace/check_generated_tasks.py +++ b/tools/bboplace/check_generated_tasks.py @@ -22,11 +22,21 @@ "candidate_limit": 1, "submission_path": "/app/solution.json", "required_text": "Submit exactly one placement for `adaptec1`", + "visible_input_copy": ( + "COPY --from=visible_input_0 " + "/opt/bboplace-bench/benchmarks/ispd2005/adaptec1 " + "/app/input/benchmarks/ispd2005/adaptec1" + ), }, "frontier-cs-2-0-bboplace-direct-iccad2015": { "candidate_limit": 1, "submission_path": "/app/solution.json", "required_text": "Submit exactly one placement for `superblue1`", + "visible_input_copy": ( + "COPY --from=visible_input_0 " + "/opt/bboplace-bench/benchmarks/iccad2015/superblue1 " + "/app/input/benchmarks/iccad2015/superblue1" + ), }, } EXPECTED_JUDGE_IMAGE = "ghcr.io/frontiercs/frontiercs-bboplace-data:2026-06-ispd-iccad" @@ -69,6 +79,7 @@ def check_task(task_dir: Path, expected: dict[str, object]) -> None: require("no GPU" in instruction, f"{task_dir.name}: missing no-GPU statement") require( f"`max_candidates_per_submission`: {expected['candidate_limit']}" in instruction + or f"max_candidates_per_submission: {expected['candidate_limit']}" in instruction or f"Only one placement is accepted" in instruction, f"{task_dir.name}: candidate limit should be {expected['candidate_limit']}", ) @@ -80,6 +91,21 @@ def check_task(task_dir: Path, expected: dict[str, object]) -> None: str(expected["submission_path"]) in submission_config, f"{task_dir.name}: generated submission path mismatch", ) + visible_input_copy = expected.get("visible_input_copy") + if visible_input_copy: + require( + "FROM ghcr.io/frontiercs/frontiercs-bboplace-data:2026-06-ispd-iccad AS visible_input_0" + in dockerfile, + f"{task_dir.name}: missing visible input image stage", + ) + require( + str(visible_input_copy) in dockerfile, + f"{task_dir.name}: missing visible input copy", + ) + require( + not (env_dir / "harbor_app" / "input").exists(), + f"{task_dir.name}: generated context should not include raw visible input", + ) def main(argv: list[str]) -> int: diff --git a/tools/bboplace/data_manifest.json b/tools/bboplace/data_manifest.json index 4ea51214..4ed75a78 100644 --- a/tools/bboplace/data_manifest.json +++ b/tools/bboplace/data_manifest.json @@ -34,49 +34,13 @@ "bigblue1", "bigblue3" ], - "extracted_size_observed": "967 MiB", - "archives": [ - { - "name": "adaptec1.tar.gz", - "url": "https://www.ispd.cc/contests/05/ispd05dp.tarballs/adaptec1.tar.gz", - "sha256": "b694dedfe15bffa7cb92dfbee0bc11906f5d334d211f51d82d0ac1effb6c0a08" - }, - { - "name": "adaptec2.tar.gz", - "url": "https://www.ispd.cc/contests/05/ispd05dp.tarballs/adaptec2.tar.gz", - "sha256": "e5a7bc0e343a97f3d9d3a1c871636a4b51da7f64ee71d2f04e7db295655a09a2" - }, - { - "name": "adaptec3.tar.gz", - "url": "https://www.ispd.cc/contests/05/ispd05dp.tarballs/adaptec3.tar.gz", - "sha256": "9ddc8f3040bd5d08609d2ff439484c3c8e3896edd9c73e5309477a2284f1edae" - }, - { - "name": "adaptec4.tar.gz", - "url": "https://www.ispd.cc/contests/05/ispd05dp.tarballs/adaptec4.tar.gz", - "sha256": "ca894bcf93ace5998dd393a6b6d5f240d3c695159cdc62ab055b8edf70ef46ab" - }, - { - "name": "bigblue1.tar.gz", - "url": "https://www.ispd.cc/contests/05/ispd05dp.tarballs/bigblue1.tar.gz", - "sha256": "d2a7a7df13242dc29f811dee8cbe4d8cd34bce6b502099ef176a3d5f7595a89d" - }, - { - "name": "bigblue2.tar.gz", - "url": "https://www.ispd.cc/contests/05/ispd05dp.tarballs/bigblue2.tar.gz", - "sha256": "3748e13367578b014424378fb441c89086c111770f835ef5b4afc3601e28316d" - }, - { - "name": "bigblue3.tar.gz", - "url": "https://www.ispd.cc/contests/05/ispd05dp.tarballs/bigblue3.tar.gz", - "sha256": "343d0c3a06eae195d92fded15dc8c8c6997db8dc76ff1da8d9c31e56bb927892" - }, - { - "name": "bigblue4.tar.gz", - "url": "https://www.ispd.cc/contests/05/ispd05dp.tarballs/bigblue4.tar.gz", - "sha256": "afd489f440d3323b643baab9082c8c9b40f350ef482d58e6ce52e431c66f6dba" - } - ] + "archive": { + "name": "ispd2005.zip", + "source": "BBOPlace-Bench README Google Drive folder 1MVIOZp2rihzIFK3C_4RqJs-bUv1TW2YT, file 1d5gpvDSQluWiT_O2QVEJgyFdkeyyVT7E", + "sha256": "bd8d44cc3a39070fff654c7e6573fa0d207c98435129d818c89559fd5672aa72" + }, + "archive_size_observed": "189 MiB", + "extracted_size_observed": "1.2 GiB" }, "iccad2015": { "benchmarks_in_image": [