From 01a12aff940923a00c242acfdb4ee80f1004fbc2 Mon Sep 17 00:00:00 2001 From: Shubham Agarwal Date: Thu, 7 Mar 2019 14:54:55 +0000 Subject: [PATCH 1/2] 1. Updated README; removing old links 2. Removing hard coded paths in lua 3. Shell script --- README.md | 6 ++++-- extractor.lua | 23 +++++++++++++++++++++-- extractor.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 extractor.sh diff --git a/README.md b/README.md index 482c158..1cb85b1 100644 --- a/README.md +++ b/README.md @@ -81,12 +81,14 @@ python data_utils.py -mode make_ie_data -input_path "../boxscore-data/rotowire" This will create files `roto-ie.h5`, `roto-ie.dict`, and `roto-ie.labels`. ### Evaluating Generated summaries -1. You can download the extraction models we ensemble to do the evaluation from this [link](https://drive.google.com/drive/u/1/folders/0B1ytQXPDuw7OdjBCUW50S2VIdDQ). There are six models in total, with the name pattern `*ie-ep*.t7`. Put these extraction models in the same directory as `extractor.lua`. (Note that `extractor.lua` hard-codes the paths to these saved models, so you'll need to change this if you want to substitute in new models.) +1. You can download the extraction models we ensemble to do the evaluation from this [link](https://drive.google.com/drive/u/1/folders/0B1ytQXPDuw7OdjBCUW50S2VIdDQ). There are six models in total, with the name pattern `*ie-ep*.t7`. Put these extraction models in the same directory as `extractor.lua`. (Note that `extractor.lua` hard-codes the paths to these saved models, so you'll need to change this if you want to substitute in new models.) -**Updated** extraction models can be downloaded from https://drive.google.com/drive/folders/1QKudbCwFuj1BAhpY58JstyGLZXvZ-2w-?usp=sharing +**Updated** extraction models can be downloaded from https://drive.google.com/drive/folders/1QKudbCwFuj1BAhpY58JstyGLZXvZ-2w-?usp=sharing **Updated** (with fix for number words and order of relations) extraction models can be downloaded from https://drive.google.com/open?id=1WGLCndbPte0p738a9p_WcBh9elbnHNmI +**Updated 7 March (@shubhamagarwal92)** extractor.lua can take file paths of the saved models. See extractor.sh for example. Default assumes extraction models are in the same directory as `extractor.lua` + 2. Once you've generated summaries, you can put them into a format the extraction system can consume as follows: ``` diff --git a/extractor.lua b/extractor.lua index 0f9bec5..e3262aa 100644 --- a/extractor.lua +++ b/extractor.lua @@ -31,6 +31,12 @@ cmd:option('-just_eval', false, [[just eval generations]]) cmd:option('-lstm', false, [[use a BLSTM rather than a convolutional model]]) cmd:option('-geom', false, [[average models geometrically]]) cmd:option('-test', false, [[use test data]]) +cmd:option('-convens_paths1', 'conv1-ep10-94-73' , [[path to conv net files]]) +cmd:option('-convens_paths2', 'conv2-ep10-95-71' , [[path to conv net files]]) +cmd:option('-convens_paths3', 'conv3-ep10-94-71' , [[path to conv net files]]) +cmd:option('-lstmens_paths1', 'lstm1-ep5-92-76' , [[path to conv net files]]) +cmd:option('-lstmens_paths2', 'lstm2-ep4-93-74' , [[path to conv net files]]) +cmd:option('-lstmens_paths3', 'lstm3-ep10-90-78' , [[path to conv net files]]) local opt = cmd:parse(arg) @@ -535,13 +541,24 @@ function set_up_saved_models() "blstmie-ep7-2.t7", "blstmie-ep10-3.t7"} --]] + --[[ local convens_paths = {"conv1ie-ep6-94-74.t7", "conv2ie-ep3-94-60.t7", "conv3ie-ep8-95-72.t7"} local lstmens_paths = {"blstm1ie-ep4-93-75.t7", "blstm2ie-ep3-93-71.t7", - "blstm3ie-ep2-94-72.t7"} + "blstm3ie-ep2-94-72.t7"} + --]] + + -- New paths taken from opt. Removing hard coding + local convens_paths = {opt.convens_paths1, + opt.convens_paths2, + opt.convens_paths3} + local lstmens_paths = {opt.lstmens_paths1, + opt.lstmens_paths2, + opt.lstmens_paths3} + opt.embed_size = 200 opt.num_filters = 200 opt.conv_fc_layer_size = 500 @@ -552,7 +569,9 @@ end function main() torch.manualSeed(opt.seed) cutorch.manualSeed(opt.seed) - cutorch.setDevice(opt.gpuid) + device_id = cutorch.getDevice() + cutorch.setDevice(device_id) + -- cutorch.setDevice(opt.gpuid) local trbatches, valbatches, V_sizes, nlabels, pred_batches, pboxrestartidxs = prep_data(opt.batchsize) local emb_sizes = {opt.embed_size, opt.embed_size/2, opt.embed_size/2} diff --git a/extractor.sh b/extractor.sh new file mode 100644 index 0000000..c879615 --- /dev/null +++ b/extractor.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + + +#├── PROJECT_DIR +#   ├── models +# │   ├── models (MODEL_DIR) +# │   └── ie_models (LUA_MODEL_DIR) +# ├── data2text-plan-py (D2T_CODE_DIR) +# │   └── data2text-1 (LUA_DIR) +# └── boxscore-data (BOXSCORE_DIR) +# └── rotowire (ROTOWIRE_DIR) + +export LUA_DIR=${PWD} +export D2T_CODE_DIR="$(dirname "LUA_DIR")" +export PROJECT_DIR="$(dirname "D2T_CODE_DIR")" + +export BOXSCORE_DIR=$PROJECT_DIR/boxscore-data +export PREPROCESS_DIR=$BOXSCORE_DIR/preprocess +export ROTOWIRE_DIR=$BOXSCORE_DIR/rotowire + +export LUA_MODEL_DIR=$PROJECT_DIR/models/ie_models +export MODEL_DIR=$PROJECT_DIR/models/models +export OUTPUT_H5=$ROTOWIRE_DIR/roto-ie.h5 +export DIC_PREFIX=$ROTOWIRE_DIR/roto-ie +export LUA_FILE=$LUA_DIR/extractor.lua + +cd $LUA_DIR + +th $LUA_FILE \ +-datafile $OUTPUT_H5 \ +-preddata $MODEL_DIR/roto_stage2_$IDENTIFIER-beam5_gens.h5 \ +-savefile $MODEL_DIR/roto_stage2_$IDENTIFIER-beam5_gens.h5-tuples.txt \ +-dict_pfx $DIC_PREFIX \ +-just_eval \ +-convens_paths1 $LUA_MODEL_DIR/conv1-ep10-94-73 \ +-convens_paths2 $LUA_MODEL_DIR/conv2-ep10-95-71 \ +-convens_paths3 $LUA_MODEL_DIR/conv3-ep10-94-71 \ +-lstmens_paths1 $LUA_MODEL_DIR/lstm1-ep5-92-76 \ +-lstmens_paths2 $LUA_MODEL_DIR/lstm2-ep4-93-74 \ +-lstmens_paths3 $LUA_MODEL_DIR/lstm3-ep10-90-78 From 2577c2e963150fa602ad2056c9f14e87f39b6b3b Mon Sep 17 00:00:00 2001 From: Shubham Agarwal Date: Fri, 8 Mar 2019 11:16:18 +0000 Subject: [PATCH 2/2] updating opts help --- extractor.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extractor.lua b/extractor.lua index e3262aa..340fafc 100644 --- a/extractor.lua +++ b/extractor.lua @@ -31,12 +31,12 @@ cmd:option('-just_eval', false, [[just eval generations]]) cmd:option('-lstm', false, [[use a BLSTM rather than a convolutional model]]) cmd:option('-geom', false, [[average models geometrically]]) cmd:option('-test', false, [[use test data]]) -cmd:option('-convens_paths1', 'conv1-ep10-94-73' , [[path to conv net files]]) -cmd:option('-convens_paths2', 'conv2-ep10-95-71' , [[path to conv net files]]) -cmd:option('-convens_paths3', 'conv3-ep10-94-71' , [[path to conv net files]]) -cmd:option('-lstmens_paths1', 'lstm1-ep5-92-76' , [[path to conv net files]]) -cmd:option('-lstmens_paths2', 'lstm2-ep4-93-74' , [[path to conv net files]]) -cmd:option('-lstmens_paths3', 'lstm3-ep10-90-78' , [[path to conv net files]]) +cmd:option('-convens_paths1', 'conv1-ep10-94-73' , [[path to conv net 1 file]]) +cmd:option('-convens_paths2', 'conv2-ep10-95-71' , [[path to conv net 2 file]]) +cmd:option('-convens_paths3', 'conv3-ep10-94-71' , [[path to conv net 3 file]]) +cmd:option('-lstmens_paths1', 'lstm1-ep5-92-76' , [[path to lstm model 1 file]]) +cmd:option('-lstmens_paths2', 'lstm2-ep4-93-74' , [[path to lstm model 2 file]]) +cmd:option('-lstmens_paths3', 'lstm3-ep10-90-78' , [[path to lstm model 3 file]]) local opt = cmd:parse(arg)