diff --git a/wikiextractor/WikiExtractor.py b/wikiextractor/WikiExtractor.py index 830235dc..ac4e4d0c 100755 --- a/wikiextractor/WikiExtractor.py +++ b/wikiextractor/WikiExtractor.py @@ -63,7 +63,7 @@ from multiprocessing import Queue, get_context, cpu_count from timeit import default_timer -from .extract import Extractor, ignoreTag, define_template, acceptedNamespaces +from wikiextractor.extract import Extractor, ignoreTag, define_template, acceptedNamespaces # =========================================================================== @@ -180,7 +180,7 @@ def open(self, filename): if self.compress: return bz2.BZ2File(filename + '.bz2', 'w') else: - return open(filename, 'w') + return open(filename, 'w', encoding = 'utf-8') # ---------------------------------------------------------------------- @@ -336,7 +336,7 @@ def collect_pages(text): def process_dump(input_file, template_file, out_file, file_size, file_compress, - process_count, html_safe, expand_templates=True): + process_count, html_safe, expand_templates=True, to_json=False): """ :param input_file: name of the wikipedia dump file; '-' to read from stdin :param template_file: optional file with template definitions. @@ -398,12 +398,11 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress, logging.info("Loaded %d templates in %.1fs", templates, template_load_elapsed) if out_file == '-': - output = sys.stdout + output = None if file_compress: logging.warn("writing to stdout, so no output compression (use an external tool)") else: - nextFile = NextFile(out_file) - output = OutputSplitter(nextFile, file_size, file_compress) + output = (out_file, file_size, file_compress) # process pages logging.info("Starting page extraction from %s.", input_file) @@ -414,7 +413,7 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress, # - a reduce process collects the results, sort them and print them. # fixes MacOS error: TypeError: cannot pickle '_io.TextIOWrapper' object - Process = get_context("fork").Process + Process = get_context("spawn").Process maxsize = 10 * process_count # output queue @@ -432,7 +431,7 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress, workers = [] for _ in range(max(1, process_count)): extractor = Process(target=extract_process, - args=(jobs_queue, output_queue, html_safe)) + args=(jobs_queue, output_queue, html_safe, to_json)) extractor.daemon = True # only live while parent process lives extractor.start() workers.append(extractor) @@ -462,8 +461,6 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress, # wait for it to finish reduce.join() - if output != sys.stdout: - output.close() extract_duration = default_timer() - extract_start extract_rate = ordinal / extract_duration logging.info("Finished %d-process extraction of %d articles in %.1fs (%.1f art/s)", @@ -474,17 +471,17 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress, # Multiprocess support -def extract_process(jobs_queue, output_queue, html_safe): +def extract_process(jobs_queue, output_queue, html_safe, to_json): """Pull tuples of raw page content, do CPU/regex-heavy fixup, push finished text :param jobs_queue: where to get jobs. :param output_queue: where to queue extracted text for output. :html_safe: whether to convert entities in text to HTML. """ while True: - job = jobs_queue.get() # job is (id, revid, urlbase, title, page) + job = jobs_queue.get() # job is (id, revid, urlbase, title, page, ordinal) if job: out = StringIO() # memory buffer - Extractor(*job[:-1]).extract(out, html_safe) # (id, urlbase, title, page) + Extractor(*job[:-1], to_json=to_json).extract(out, html_safe) text = out.getvalue() output_queue.put((job[-1], text)) # (ordinal, extracted_text) out.close() @@ -492,13 +489,19 @@ def extract_process(jobs_queue, output_queue, html_safe): break -def reduce_process(output_queue, output): +def reduce_process(output_queue, output_file=None): """ Pull finished article text, write series of files (or stdout) :param output_queue: text to be output. - :param output: file object where to print. + :param output: A optional tuple that consists of output_file, file_size and file_compress argument. """ + if output_file is None: + output = sys.stdout # default + else: + nextFile, file_size, file_compress = output_file + output = OutputSplitter(NextFile(nextFile), file_size, file_compress) + interval_start = default_timer() period = 100000 # FIXME: use a heap @@ -522,6 +525,9 @@ def reduce_process(output_queue, output): ordinal, text = pair ordering_buffer[ordinal] = text + if output != sys.stdout: + output.close() + # ---------------------------------------------------------------------- @@ -637,7 +643,7 @@ def main(): return process_dump(input_file, args.templates, output_path, file_size, - args.compress, args.processes, args.html_safe, not args.no_templates) + args.compress, args.processes, args.html_safe, not args.no_templates, args.json) if __name__ == '__main__': main() diff --git a/wikiextractor/extract.py b/wikiextractor/extract.py index a00e23d2..7d55d3fe 100644 --- a/wikiextractor/extract.py +++ b/wikiextractor/extract.py @@ -30,7 +30,7 @@ # ---------------------------------------------------------------------- # match tail after wikilink -tailRE = re.compile('\w+') +tailRE = re.compile(r'\w+') syntaxhighlight = re.compile('<syntaxhighlight .*?>(.*?)</syntaxhighlight>', re.DOTALL) ## PARAMS #################################################################### @@ -168,8 +168,8 @@ def clean(extractor, text, expand_templates=False, html_safe=True): text = text.replace('\t', ' ') text = spaces.sub(' ', text) text = dots.sub('...', text) - text = re.sub(u' (,:\.\)\]»)', r'\1', text) - text = re.sub(u'(\[\(«) ', r'\1', text) + text = re.sub(r' (,:\.\)\]»)', r'\1', text) + text = re.sub(r'(\[\(«) ', r'\1', text) text = re.sub(r'\n\W+?\n', '\n', text, flags=re.U) # lines with only punctuations text = text.replace(',,', ',').replace(',.', '.') if html_safe: @@ -259,7 +259,8 @@ def compact(text, mark_headers=False): line = line[l:].strip() page.append(listItem[type] % line) else: - continue + if not line.lstrip('*#;').lower().startswith('redirect'): + page.append(line.lstrip('*#;')) # fix for bullet points elif len(listLevel): # implies Extractor.HtmlFormatting for c in reversed(listLevel): page.append(listClose[c]) @@ -380,12 +381,12 @@ def dropSpans(spans, text): # as well as U+3000 is IDEOGRAPHIC SPACE for bug 19052 EXT_LINK_URL_CLASS = r'[^][<>"\x00-\x20\x7F\s]' ExtLinkBracketedRegex = re.compile( - '\[(((?i)' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]', - re.S | re.U) + r'\[((?:' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]', + re.S | re.U | re.I) EXT_IMAGE_REGEX = re.compile( r"""^(http://|https://)([^][<>"\x00-\x20\x7F\s]+) - /([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""", - re.X | re.S | re.U) + /([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.(gif|png|jpg|jpeg)$""", + re.X | re.S | re.U | re.I) def replaceExternalLinks(text): @@ -396,7 +397,7 @@ def replaceExternalLinks(text): cur = m.end() url = m.group(1) - label = m.group(3) + label = m.group(2) # # The characters '<' and '>' (which were escaped by # # removeHTMLtags()) should not be included in @@ -734,7 +735,7 @@ def fixup(m): except: return text # leave as is - return re.sub("&#?(\w+);", fixup, text) + return re.sub(r"&#?(\w+);", fixup, text) # Match HTML comments @@ -921,13 +922,13 @@ class Extractor(): ## # Whether to produce json instead of the default output format. - toJson = False + to_json = False ## # Obtained from TemplateNamespace templatePrefix = '' - def __init__(self, id, revid, urlbase, title, page): + def __init__(self, id, revid, urlbase, title, page, to_json = False): """ :param page: a list of lines. """ @@ -942,6 +943,7 @@ def __init__(self, id, revid, urlbase, title, page): self.recursion_exceeded_2_errs = 0 # template recursion within expandTemplate() self.recursion_exceeded_3_errs = 0 # parameter recursion self.template_title_errs = 0 + self.to_json = to_json # fixed def clean_text(self, text, mark_headers=False, expand_templates=True, html_safe=True): @@ -1394,11 +1396,11 @@ def findMatchingBraces(text, ldelim=0): # {{{link|{{ucfirst:{{{1}}}}}} interchange}}} if ldelim: # 2-3 - reOpen = re.compile('[{]{%d,}' % ldelim) # at least ldelim - reNext = re.compile('[{]{2,}|}{2,}') # at least 2 open or close bracces + reOpen = re.compile(r'[{]{%d,}' % ldelim) # at least ldelim + reNext = re.compile(r'[{]{2,}|}{2,}') # at least 2 open or close bracces else: - reOpen = re.compile('{{2,}|\[{2,}') - reNext = re.compile('{{2,}|}{2,}|\[{2,}|]{2,}') # at least 2 + reOpen = re.compile(r'{{2,}|\[{2,}') + reNext = re.compile(r'{{2,}|}{2,}|\[{2,}|]{2,}') # at least 2 cur = 0 while True: @@ -1649,7 +1651,7 @@ def sharp_ifeq(lvalue, rvalue, valueIfTrue, valueIfFalse=None, *args): def sharp_iferror(test, then='', Else=None, *args): - if re.match('<(?:strong|span|p|div)\s(?:[^\s>]*\s+)*?class="(?:[^"\s>]*\s+)*?error(?:\s[^">]*)?"', test): + if re.match(r'<(?:strong|span|p|div)\s(?:[^\s>]*\s+)*?class="(?:[^"\s>]*\s+)*?error(?:\s[^">]*)?"', test): return then elif Else is None: return test.strip() @@ -1820,7 +1822,7 @@ def define_template(title, page): # title = normalizeTitle(title) # check for redirects - m = re.match('#REDIRECT.*?\[\[([^\]]*)]]', page[0], re.IGNORECASE) + m = re.match(r'#REDIRECT.*?\[\[([^\]]*)]]', page[0], re.IGNORECASE) if m: redirects[title] = m.group(1) # normalizeTitle(m.group(1)) return