From f0efe57665e9415bd8ce21fc22b8ba5a0c4ae50e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 Jan 2018 12:55:49 +0100 Subject: [PATCH 1/3] copy msgextractor dep --- .travis.yml | 5 +- fulltext/backends/__msg.py | 2 +- fulltext/data/winmake.py | 3 - fulltext/msgextractor.py | 550 +++++++++++++++++++++++++++++++++++++ requirements.txt | 2 - 5 files changed, 552 insertions(+), 10 deletions(-) create mode 100644 fulltext/msgextractor.py diff --git a/.travis.yml b/.travis.yml index 034a92f..2cee4b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,6 @@ language: python python: - # XXX - commented because can't figure out how to install ExtractMsg - # on Python 2.7: - # https://travis-ci.org/btimby/fulltext/jobs/329810555 - # - "2.7" + - "2.7" - "3.5" before_install: - make sysdeps diff --git a/fulltext/backends/__msg.py b/fulltext/backends/__msg.py index 41ea3a7..0d86d35 100644 --- a/fulltext/backends/__msg.py +++ b/fulltext/backends/__msg.py @@ -1,7 +1,7 @@ import contextlib from six import StringIO -from ExtractMsg import Message +from fulltext.msgextractor import Message from fulltext.util import BaseBackend diff --git a/fulltext/data/winmake.py b/fulltext/data/winmake.py index 5073a0b..ca4b49e 100644 --- a/fulltext/data/winmake.py +++ b/fulltext/data/winmake.py @@ -329,9 +329,6 @@ def assertMultiLineEqual(a, b): def install_deps(): sh("venv\\Scripts\\python -m pip install pyinstaller pypiwin32") - sh("venv\\Scripts\\python -m pip install " - "https://github.com/mattgwwalker/msg-extractor/zipball/" - "master#egg=ExtractMsg") sh("venv\\Scripts\\python setup.py install") def run_pyinstaller(): diff --git a/fulltext/msgextractor.py b/fulltext/msgextractor.py new file mode 100644 index 0000000..7b7f942 --- /dev/null +++ b/fulltext/msgextractor.py @@ -0,0 +1,550 @@ +#!/usr/bin/env python +# -*- coding: latin-1 -*- +""" +ExtractMsg: + Extracts emails and attachments saved in Microsoft Outlook's .msg files + +Note: this is copied from: +https://github.com/mattgwwalker/msg-extractor +""" + +__author__ = "Matthew Walker" +__date__ = "2016-10-09" +__version__ = '0.3' + +# --- LICENSE ----------------------------------------------------------------- +# +# Copyright 2013 Matthew Walker +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +import glob +import traceback +from email.parser import Parser as EmailParser +import email.utils +import olefile as OleFile + +# This property information was sourced from +# http://www.fileformat.info/format/outlookmsg/index.htm +# on 2013-07-22. +properties = { + '001A': 'Message class', + '0037': 'Subject', + '003D': 'Subject prefix', + '0040': 'Received by name', + '0042': 'Sent repr name', + '0044': 'Rcvd repr name', + '004D': 'Org author name', + '0050': 'Reply rcipnt names', + '005A': 'Org sender name', + '0064': 'Sent repr adrtype', + '0065': 'Sent repr email', + '0070': 'Topic', + '0075': 'Rcvd by adrtype', + '0076': 'Rcvd by email', + '0077': 'Repr adrtype', + '0078': 'Repr email', + '007d': 'Message header', + '0C1A': 'Sender name', + '0C1E': 'Sender adr type', + '0C1F': 'Sender email', + '0E02': 'Display BCC', + '0E03': 'Display CC', + '0E04': 'Display To', + '0E1D': 'Subject (normalized)', + '0E28': 'Recvd account1 (uncertain)', + '0E29': 'Recvd account2 (uncertain)', + '1000': 'Message body', + '1008': 'RTF sync body tag', + '1035': 'Message ID (uncertain)', + '1046': 'Sender email (uncertain)', + '3001': 'Display name', + '3002': 'Address type', + '3003': 'Email address', + '39FE': '7-bit email (uncertain)', + '39FF': '7-bit display name', + + # Attachments (37xx) + '3701': 'Attachment data', + '3703': 'Attachment extension', + '3704': 'Attachment short filename', + '3707': 'Attachment long filename', + '370E': 'Attachment mime tag', + '3712': 'Attachment ID (uncertain)', + + # Address book (3Axx): + '3A00': 'Account', + '3A02': 'Callback phone no', + '3A05': 'Generation', + '3A06': 'Given name', + '3A08': 'Business phone', + '3A09': 'Home phone', + '3A0A': 'Initials', + '3A0B': 'Keyword', + '3A0C': 'Language', + '3A0D': 'Location', + '3A11': 'Surname', + '3A15': 'Postal address', + '3A16': 'Company name', + '3A17': 'Title', + '3A18': 'Department', + '3A19': 'Office location', + '3A1A': 'Primary phone', + '3A1B': 'Business phone 2', + '3A1C': 'Mobile phone', + '3A1D': 'Radio phone no', + '3A1E': 'Car phone no', + '3A1F': 'Other phone', + '3A20': 'Transmit dispname', + '3A21': 'Pager', + '3A22': 'User certificate', + '3A23': 'Primary Fax', + '3A24': 'Business Fax', + '3A25': 'Home Fax', + '3A26': 'Country', + '3A27': 'Locality', + '3A28': 'State/Province', + '3A29': 'Street address', + '3A2A': 'Postal Code', + '3A2B': 'Post Office Box', + '3A2C': 'Telex', + '3A2D': 'ISDN', + '3A2E': 'Assistant phone', + '3A2F': 'Home phone 2', + '3A30': 'Assistant', + '3A44': 'Middle name', + '3A45': 'Dispname prefix', + '3A46': 'Profession', + '3A48': 'Spouse name', + '3A4B': 'TTYTTD radio phone', + '3A4C': 'FTP site', + '3A4E': 'Manager name', + '3A4F': 'Nickname', + '3A51': 'Business homepage', + '3A57': 'Company main phone', + '3A58': 'Childrens names', + '3A59': 'Home City', + '3A5A': 'Home Country', + '3A5B': 'Home Postal Code', + '3A5C': 'Home State/Provnce', + '3A5D': 'Home Street', + '3A5F': 'Other adr City', + '3A60': 'Other adr Country', + '3A61': 'Other adr PostCode', + '3A62': 'Other adr Province', + '3A63': 'Other adr Street', + '3A64': 'Other adr PO box', + + '3FF7': 'Server (uncertain)', + '3FF8': 'Creator1 (uncertain)', + '3FFA': 'Creator2 (uncertain)', + '3FFC': 'To email (uncertain)', + '403D': 'To adrtype (uncertain)', + '403E': 'To email (uncertain)', + '5FF6': 'To (uncertain)'} + + +def windowsUnicode(string): + if string is None: + return None + if sys.version_info[0] >= 3: # Python 3 + return str(string, 'utf_16_le') + else: # Python 2 + return unicode(string, 'utf_16_le') # NOQA + + +class Attachment: + def __init__(self, msg, dir_): + # Get long filename + self.longFilename = msg._getStringStream([dir_, '__substg1.0_3707']) + + # Get short filename + self.shortFilename = msg._getStringStream([dir_, '__substg1.0_3704']) + + # Get attachment data + self.data = msg._getStream([dir_, '__substg1.0_37010102']) + + def save(self): + # Use long filename as first preference + filename = self.longFilename + # Otherwise use the short filename + if filename is None: + filename = self.shortFilename + # Otherwise just make something up! + if filename is None: + import random + import string + filename = 'UnknownFilename ' + \ + ''.join(random.choice(string.ascii_uppercase + string.digits) + for _ in range(5)) + ".bin" + f = open(filename, 'wb') + f.write(self.data) + f.close() + return filename + + +class Message(OleFile.OleFileIO): + def __init__(self, filename): + OleFile.OleFileIO.__init__(self, filename) + + def _getStream(self, filename): + if self.exists(filename): + stream = self.openstream(filename) + return stream.read() + else: + return None + + def _getStringStream(self, filename, prefer='unicode'): + """Gets a string representation of the requested filename. + Checks for both ASCII and Unicode representations and returns + a value if possible. If there are both ASCII and Unicode + versions, then the parameter /prefer/ specifies which will be + returned. + """ + + if isinstance(filename, list): + # Join with slashes to make it easier to append the type + filename = "/".join(filename) + + asciiVersion = self._getStream(filename + '001E') + unicodeVersion = windowsUnicode(self._getStream(filename + '001F')) + if asciiVersion is None: + return unicodeVersion + elif unicodeVersion is None: + return asciiVersion + else: + if prefer == 'unicode': + return unicodeVersion + else: + return asciiVersion + + @property + def subject(self): + return self._getStringStream('__substg1.0_0037') + + @property + def header(self): + try: + return self._header + except Exception: + headerText = self._getStringStream('__substg1.0_007D') + if headerText is not None: + self._header = EmailParser().parsestr(headerText) + else: + self._header = None + return self._header + + @property + def date(self): + # Get the message's header and extract the date + if self.header is None: + return None + else: + return self.header['date'] + + @property + def parsedDate(self): + return email.utils.parsedate(self.date) + + @property + def sender(self): + try: + return self._sender + except Exception: + # Check header first + if self.header is not None: + headerResult = self.header["from"] + if headerResult is not None: + self._sender = headerResult + return headerResult + + # Extract from other fields + text = self._getStringStream('__substg1.0_0C1A') + email = self._getStringStream('__substg1.0_0C1F') + result = None + if text is None: + result = email + else: + result = text + if email is not None: + result = result + " <" + email + ">" + + self._sender = result + return result + + @property + def to(self): + try: + return self._to + except Exception: + # Check header first + if self.header is not None: + headerResult = self.header["to"] + if headerResult is not None: + self._to = headerResult + return headerResult + + # Extract from other fields + # TODO: This should really extract data from the recip folders, + # but how do you know which is to/cc/bcc? + display = self._getStringStream('__substg1.0_0E04') + self._to = display + return display + + @property + def cc(self): + try: + return self._cc + except Exception: + # Check header first + if self.header is not None: + headerResult = self.header["cc"] + if headerResult is not None: + self._cc = headerResult + return headerResult + + # Extract from other fields + # TODO: This should really extract data from the recip folders, + # but how do you know which is to/cc/bcc? + display = self._getStringStream('__substg1.0_0E03') + self._cc = display + return display + + @property + def body(self): + # Get the message body + return self._getStringStream('__substg1.0_1000') + + @property + def attachments(self): + try: + return self._attachments + except Exception: + # Get the attachments + attachmentDirs = [] + + for dir_ in self.listdir(): + if dir_[0].startswith('__attach') and \ + dir_[0] not in attachmentDirs: + attachmentDirs.append(dir_[0]) + + self._attachments = [] + + for attachmentDir in attachmentDirs: + self._attachments.append(Attachment(self, attachmentDir)) + + return self._attachments + + def save(self, toJson=False, useFileName=False, raw=False): + '''Saves the message body and attachments found in the message. + Setting toJson to true will output the message body as + JSON-formatted text. The body and attachments are stored in a folder. + Setting useFileName to true will mean that the filename is used as + the name of the folder; otherwise, the message's date + and subject are used as the folder name. + ''' + + if useFileName: + # strip out the extension + dirName = filename.split('/').pop().split('.')[0] + else: + # Create a directory based on the date and subject of the message + d = self.parsedDate + if d is not None: + dirName = '{0:02d}-{1:02d}-{2:02d}_{3:02d}{4:02d}'.format(*d) + else: + dirName = "UnknownDate" + + if self.subject is None: + subject = "[No subject]" + else: + subject = "".join( + i for i in self.subject if i not in r'\/:*?"<>|') + + dirName = dirName + " " + subject + + def addNumToDir(dirName): + # Attempt to create the directory with a '(n)' appended + + for i in range(2, 100): + try: + newDirName = dirName + " (" + str(i) + ")" + os.makedirs(newDirName) + return newDirName + except Exception: + pass + return None + + try: + os.makedirs(dirName) + except Exception: + newDirName = addNumToDir(dirName) + if newDirName is not None: + dirName = newDirName + else: + raise Exception( + "Failed to create directory '%s'. Does it already exist?" % + dirName) + + oldDir = os.getcwd() + try: + os.chdir(dirName) + + # Save the message body + fext = 'json' if toJson else 'text' + f = open("message." + fext, "w") + # From, to , cc, subject, date + + def xstr(s): + return '' if s is None else str(s) + + attachmentNames = [] + # Save the attachments + for attachment in self.attachments: + attachmentNames.append(attachment.save()) + + if toJson: + import json + from imapclient.imapclient import decode_utf7 + + emailObj = {'from': xstr(self.sender), + 'to': xstr(self.to), + 'cc': xstr(self.cc), + 'subject': xstr(self.subject), + 'date': xstr(self.date), + 'attachments': attachmentNames, + 'body': decode_utf7(self.body)} + + f.write(json.dumps(emailObj, ensure_ascii=True)) + else: + f.write("From: " + xstr(self.sender) + "\n") + f.write("To: " + xstr(self.to) + "\n") + f.write("CC: " + xstr(self.cc) + "\n") + f.write("Subject: " + xstr(self.subject) + "\n") + f.write("Date: " + xstr(self.date) + "\n") + f.write("-----------------\n\n") + f.write(self.body) + + f.close() + + except Exception: + self.saveRaw() + raise + + finally: + # Return to previous directory + os.chdir(oldDir) + + def saveRaw(self): + # Create a 'raw' folder + oldDir = os.getcwd() + try: + rawDir = "raw" + os.makedirs(rawDir) + os.chdir(rawDir) + sysRawDir = os.getcwd() + + # Loop through all the directories + for dir_ in self.listdir(): + sysdir = "/".join(dir_) + code = dir_[-1][-8:-4] + global properties + if code in properties: + sysdir = sysdir + " - " + properties[code] + os.makedirs(sysdir) + os.chdir(sysdir) + + # Generate appropriate filename + if dir_[-1].endswith("001E"): + filename = "contents.txt" + else: + filename = "contents" + + # Save contents of directory + f = open(filename, 'wb') + f.write(self._getStream(dir_)) + f.close() + + # Return to base directory + os.chdir(sysRawDir) + + finally: + os.chdir(oldDir) + + def dump(self): + # Prints out a summary of the message + print('Message') + print('Subject:', self.subject) + print('Date:', self.date) + print('Body:') + print(self.body) + + def debug(self): + for dir_ in self.listdir(): + if dir_[-1].endswith('001E'): # FIXME: Check for unicode 001F too + print("Directory: " + str(dir)) + print("Contents: " + self._getStream(dir)) + + def save_attachments(self, raw=False): + """Saves only attachments in the same folder. + """ + for attachment in self.attachments: + attachment.save() + + +if __name__ == "__main__": + if len(sys.argv) <= 1: + print(__doc__) + print(""" +Launched from command line, this script parses Microsoft Outlook Message files +and save their contents to the current directory. On error the script will +write out a 'raw' directory will all the details from the file, but in a +less-than-desirable format. To force this mode, the flag '--raw' +can be specified. + +Usage: [file2 ...] + or: --raw + or: --json + + to name the directory as the .msg file, --use-file-name +""") + sys.exit() + + writeRaw = False + toJson = False + useFileName = False + + for rawFilename in sys.argv[1:]: + if rawFilename == '--raw': + writeRaw = True + + if rawFilename == '--json': + toJson = True + + if rawFilename == '--use-file-name': + useFileName = True + + for filename in glob.glob(rawFilename): + msg = Message(filename) + try: + if writeRaw: + msg.saveRaw() + else: + msg.save(toJson, useFileName) + except Exception: + # msg.debug() + print("Error with file '" + filename + "': " + + traceback.format_exc()) diff --git a/requirements.txt b/requirements.txt index 202dbe3..1507b09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,4 @@ mock rarfile flake8 -git+https://github.com/mattgwwalker/msg-extractor.git@2a24c9950d34932ed5979693cb70d758d78715df#egg=ExtractMsg - git+https://github.com/smarnach/pyexiftool.git#egg=pyexiftool From 4ad62109f6c8a643ec10782f56a051d3cd6fa117 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 Jan 2018 13:00:55 +0100 Subject: [PATCH 2/3] add olefile dep for msgextrractor lib --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 1507b09..008b9ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,6 @@ contextlib2 mock rarfile flake8 +olefile git+https://github.com/smarnach/pyexiftool.git#egg=pyexiftool From d7638b8723b31ac22d543aed327df8c5f069ed75 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 Jan 2018 13:08:59 +0100 Subject: [PATCH 3/3] install pkg in Makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 8f3575d..6ef1a20 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ test: ## Run tests. ci: ## Run CI tests. ${MAKE} sysdeps ${MAKE} pydeps + ${MAKE} install ${MAKE} test ${MAKE} lint