diff --git a/README.md b/README.md index 707807d..3f86f7b 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,39 @@ * **Metrics:** All metric-related code can be found [here](./cik_benchmark/metrics). * **Experiments:** Code used to run the experiments can be found [here](./experiments). +## Getting started + +Once the repository has been cloned locally, start by creating a conda package: +```bash +conda create --name CiK python=3.11 +conda activate CiK +``` +(Note that there are compatibility issues with very recent versions of Python, +so we recommend using Python 3.11.) +Next, install the package: +```bash +pip install -e . +``` +The `-e` option is useful for development, as it installs the package in +editable mode. This means that changes to the code will be reflected in the +package without needing to reinstall it. In addition, if you intend on running +the R baselines, you will need to install additional packages. You also +separately need to make sure that R is installed on your system, which you can +find how to do at the [R Project](https://r-project.org) website. +```bash +pip install -r ./requirements-r.txt +``` + +You will need to set some environment variables; the full list appears in the +next section. In particular, `NIXTLA_API_KEY` must be set for the code to +baselines to run. Go to [](https://dashboard.nixtla.io/) to set up an account +and get an API key. + +When all is done, you're ready to run baseline models. For instance, to run the +Qwen 7B-parameter model locally (it will be downloaded automatically from Hugging Face the first time you run it), you can run: +```bash +python run_baselines.py --exp-spec experiments/direct-prompt-models/qwen_7b_instruct_ctx_g2.json +``` ## Setting environment variables diff --git a/cik_benchmark/baselines/direct_prompt.py b/cik_benchmark/baselines/direct_prompt.py index 9e83258..756a125 100644 --- a/cik_benchmark/baselines/direct_prompt.py +++ b/cik_benchmark/baselines/direct_prompt.py @@ -89,25 +89,6 @@ def huggingface_instruct_model_client( future_timestamps=None, **kwargs, ): - if constrained_decoding: - assert ( - future_timestamps is not None - ), "Future timestamps must be provided for constrained decoding" - - def constrained_decoding_regex(required_timestamps): - """ - Generates a regular expression to force the model output - to satisfy the required format and provide values for - all required timestamps - - """ - timestamp_regex = "".join( - [ - r"\(\s*{}\s*,\s*[-+]?\d+(\.\d+)?\)\n".format(re.escape(ts)) - for ts in required_timestamps - ] - ) - return r"\n{}<\/forecast>".format(timestamp_regex) # Make generation pipeline pipe = pipeline( @@ -117,11 +98,34 @@ def constrained_decoding_regex(required_timestamps): device_map="auto", ) - # Build a regex parser with the generated regex - parser = RegexParser(constrained_decoding_regex(future_timestamps)) - prefix_function = build_transformers_prefix_allowed_tokens_fn( - pipe.tokenizer, parser - ) + # If constrained decoding, build a prefix function. This requires the future + # timestamps to be specified, so we can make them part of the regex + prefix_function = None + if constrained_decoding: + assert ( + future_timestamps is not None + ), "Future timestamps must be provided for constrained decoding" + + def constrained_decoding_regex(required_timestamps): + """ + Generates a regular expression to force the model output + to satisfy the required format and provide values for + all required timestamps + + """ + timestamp_regex = "".join( + [ + r"\(\s*{}\s*,\s*[-+]?\d+(\.\d+)?\)\n".format(re.escape(ts)) + for ts in required_timestamps + ] + ) + return r"\n{}<\/forecast>".format(timestamp_regex) + + # Build a regex parser with the generated regex + parser = RegexParser(constrained_decoding_regex(future_timestamps)) + prefix_function = build_transformers_prefix_allowed_tokens_fn( + pipe.tokenizer, parser + ) # Now extract the assistant's reply choices = [] diff --git a/experiments/direct-prompt-models/qwen_1.5b_instruct_ctx_g2.json b/experiments/direct-prompt-models/qwen_1.5b_instruct_ctx_g2.json new file mode 100644 index 0000000..82aebd2 --- /dev/null +++ b/experiments/direct-prompt-models/qwen_1.5b_instruct_ctx_g2.json @@ -0,0 +1,4 @@ +[ + {"label": "CC-Qwen-2.5-1.5B-Instruct (ctx)", "method": "directprompt", "llm": "qwen2.5-1.5B-Instruct", "use_context": true, "temperature": 1.0, + "batch_size_on_retry":10, "batch_size":10, "n_retries": 10} +] diff --git a/run_baselines.py b/run_baselines.py index 69b8502..3541d99 100644 --- a/run_baselines.py +++ b/run_baselines.py @@ -217,12 +217,13 @@ def experiment_directprompt( "gemma-2-27B-instruct": {"input": 0.0, "output": 0.0}, # Toolkit } if not llm.startswith("openrouter-") and llm not in openai_costs: - raise ValueError(f"Invalid model: {llm} -- Not in cost dictionary") + # Log a warning if the model is not in the cost dictionary + logging.warning(f"Model {llm} not in cost dictionary; assuming zero token cost") dp_forecaster = DirectPrompt( model=llm, use_context=use_context, - token_cost=openai_costs[llm] if not llm.startswith("openrouter-") else None, + token_cost=None if llm.startswith("openrouter-") else openai_costs.get(llm, None), batch_size=batch_size, batch_size_on_retry=batch_size_on_retry, n_retries=n_retries,