Skip to content

Fix compatibility with Python 3.13 and Windows, enable --json flag su… #343

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
OlaszPL wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix compatibility with Python 3.13 and Windows, enable --json flag su… #343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4256072
Fix compatibility with Python 3.13 and Windows, enable --json flag su…
OlaszPL May 1, 2025
33eac0b
Refactor extract_process
OlaszPL May 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 22 additions & 16 deletions wikiextractor/WikiExtractor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
from multiprocessing import Queue, get_context, cpu_count
from timeit import default_timer

from .extract import Extractor, ignoreTag, define_template, acceptedNamespaces
from wikiextractor.extract import Extractor, ignoreTag, define_template, acceptedNamespaces

# ===========================================================================

Expand Down Expand Up @@ -180,7 +180,7 @@ def open(self, filename):
if self.compress:
return bz2.BZ2File(filename + '.bz2', 'w')
else:
return open(filename, 'w')
return open(filename, 'w', encoding = 'utf-8')


# ----------------------------------------------------------------------
Expand Down Expand Up @@ -336,7 +336,7 @@ def collect_pages(text):


def process_dump(input_file, template_file, out_file, file_size, file_compress,
process_count, html_safe, expand_templates=True):
process_count, html_safe, expand_templates=True, to_json=False):
"""
:param input_file: name of the wikipedia dump file; '-' to read from stdin
:param template_file: optional file with template definitions.
Expand Down Expand Up @@ -398,12 +398,11 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress,
logging.info("Loaded %d templates in %.1fs", templates, template_load_elapsed)

if out_file == '-':
output = sys.stdout
output = None
if file_compress:
logging.warn("writing to stdout, so no output compression (use an external tool)")
else:
nextFile = NextFile(out_file)
output = OutputSplitter(nextFile, file_size, file_compress)
output = (out_file, file_size, file_compress)

# process pages
logging.info("Starting page extraction from %s.", input_file)
Expand All @@ -414,7 +413,7 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress,
# - a reduce process collects the results, sort them and print them.

# fixes MacOS error: TypeError: cannot pickle '_io.TextIOWrapper' object
Process = get_context("fork").Process
Process = get_context("spawn").Process

maxsize = 10 * process_count
# output queue
Expand All @@ -432,7 +431,7 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress,
workers = []
for _ in range(max(1, process_count)):
extractor = Process(target=extract_process,
args=(jobs_queue, output_queue, html_safe))
args=(jobs_queue, output_queue, html_safe, to_json))
extractor.daemon = True # only live while parent process lives
extractor.start()
workers.append(extractor)
Expand Down Expand Up @@ -462,8 +461,6 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress,
# wait for it to finish
reduce.join()

if output != sys.stdout:
output.close()
extract_duration = default_timer() - extract_start
extract_rate = ordinal / extract_duration
logging.info("Finished %d-process extraction of %d articles in %.1fs (%.1f art/s)",
Expand All @@ -474,31 +471,37 @@ def process_dump(input_file, template_file, out_file, file_size, file_compress,
# Multiprocess support


def extract_process(jobs_queue, output_queue, html_safe):
def extract_process(jobs_queue, output_queue, html_safe, to_json):
"""Pull tuples of raw page content, do CPU/regex-heavy fixup, push finished text
:param jobs_queue: where to get jobs.
:param output_queue: where to queue extracted text for output.
:html_safe: whether to convert entities in text to HTML.
"""
while True:
job = jobs_queue.get() # job is (id, revid, urlbase, title, page)
job = jobs_queue.get() # job is (id, revid, urlbase, title, page, ordinal)
if job:
out = StringIO() # memory buffer
Extractor(*job[:-1]).extract(out, html_safe) # (id, urlbase, title, page)
Extractor(*job[:-1], to_json=to_json).extract(out, html_safe)
text = out.getvalue()
output_queue.put((job[-1], text)) # (ordinal, extracted_text)
out.close()
else:
break


def reduce_process(output_queue, output):
def reduce_process(output_queue, output_file=None):
"""
Pull finished article text, write series of files (or stdout)
:param output_queue: text to be output.
:param output: file object where to print.
:param output: A optional tuple that consists of output_file, file_size and file_compress argument.
"""

if output_file is None:
output = sys.stdout # default
else:
nextFile, file_size, file_compress = output_file
output = OutputSplitter(NextFile(nextFile), file_size, file_compress)

interval_start = default_timer()
period = 100000
# FIXME: use a heap
Expand All @@ -522,6 +525,9 @@ def reduce_process(output_queue, output):
ordinal, text = pair
ordering_buffer[ordinal] = text

if output != sys.stdout:
output.close()


# ----------------------------------------------------------------------

Expand Down Expand Up @@ -637,7 +643,7 @@ def main():
return

process_dump(input_file, args.templates, output_path, file_size,
args.compress, args.processes, args.html_safe, not args.no_templates)
args.compress, args.processes, args.html_safe, not args.no_templates, args.json)

if __name__ == '__main__':
main()
38 changes: 20 additions & 18 deletions wikiextractor/extract.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
# ----------------------------------------------------------------------

# match tail after wikilink
tailRE = re.compile('\w+')
tailRE = re.compile(r'\w+')
syntaxhighlight = re.compile('<syntaxhighlight .*?>(.*?)</syntaxhighlight>', re.DOTALL)

## PARAMS ####################################################################
Expand Down Expand Up @@ -168,8 +168,8 @@ def clean(extractor, text, expand_templates=False, html_safe=True):
text = text.replace('\t', ' ')
text = spaces.sub(' ', text)
text = dots.sub('...', text)
text = re.sub(u' (,:\.\)\]»)', r'\1', text)
text = re.sub(u'(\[\(«) ', r'\1', text)
text = re.sub(r' (,:\.\)\]»)', r'\1', text)
text = re.sub(r'(\[\(«) ', r'\1', text)
text = re.sub(r'\n\W+?\n', '\n', text, flags=re.U) # lines with only punctuations
text = text.replace(',,', ',').replace(',.', '.')
if html_safe:
Expand Down Expand Up @@ -259,7 +259,8 @@ def compact(text, mark_headers=False):
line = line[l:].strip()
page.append(listItem[type] % line)
else:
continue
if not line.lstrip('*#;').lower().startswith('redirect'):
page.append(line.lstrip('*#;')) # fix for bullet points
elif len(listLevel): # implies Extractor.HtmlFormatting
for c in reversed(listLevel):
page.append(listClose[c])
Expand Down Expand Up @@ -380,12 +381,12 @@ def dropSpans(spans, text):
# as well as U+3000 is IDEOGRAPHIC SPACE for bug 19052
EXT_LINK_URL_CLASS = r'[^][<>"\x00-\x20\x7F\s]'
ExtLinkBracketedRegex = re.compile(
'\[(((?i)' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]',
re.S | re.U)
r'\[((?:' + '|'.join(wgUrlProtocols) + ')' + EXT_LINK_URL_CLASS + r'+)\s*([^\]\x00-\x08\x0a-\x1F]*?)\]',
re.S | re.U | re.I)
EXT_IMAGE_REGEX = re.compile(
r"""^(http://|https://)([^][<>"\x00-\x20\x7F\s]+)
/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.((?i)gif|png|jpg|jpeg)$""",
re.X | re.S | re.U)
/([A-Za-z0-9_.,~%\-+&;#*?!=()@\x80-\xFF]+)\.(gif|png|jpg|jpeg)$""",
re.X | re.S | re.U | re.I)


def replaceExternalLinks(text):
Expand All @@ -396,7 +397,7 @@ def replaceExternalLinks(text):
cur = m.end()

url = m.group(1)
label = m.group(3)
label = m.group(2)

# # The characters '<' and '>' (which were escaped by
# # removeHTMLtags()) should not be included in
Expand Down Expand Up @@ -734,7 +735,7 @@ def fixup(m):
except:
return text # leave as is

return re.sub("&#?(\w+);", fixup, text)
return re.sub(r"&#?(\w+);", fixup, text)


# Match HTML comments
Expand Down Expand Up @@ -921,13 +922,13 @@ class Extractor():

##
# Whether to produce json instead of the default <doc> output format.
toJson = False
to_json = False

##
# Obtained from TemplateNamespace
templatePrefix = ''

def __init__(self, id, revid, urlbase, title, page):
def __init__(self, id, revid, urlbase, title, page, to_json = False):
"""
:param page: a list of lines.
"""
Expand All @@ -942,6 +943,7 @@ def __init__(self, id, revid, urlbase, title, page):
self.recursion_exceeded_2_errs = 0 # template recursion within expandTemplate()
self.recursion_exceeded_3_errs = 0 # parameter recursion
self.template_title_errs = 0
self.to_json = to_json # fixed

def clean_text(self, text, mark_headers=False, expand_templates=True,
html_safe=True):
Expand Down Expand Up @@ -1394,11 +1396,11 @@ def findMatchingBraces(text, ldelim=0):
# {{{link|{{ucfirst:{{{1}}}}}} interchange}}}

if ldelim: # 2-3
reOpen = re.compile('[{]{%d,}' % ldelim) # at least ldelim
reNext = re.compile('[{]{2,}|}{2,}') # at least 2 open or close bracces
reOpen = re.compile(r'[{]{%d,}' % ldelim) # at least ldelim
reNext = re.compile(r'[{]{2,}|}{2,}') # at least 2 open or close bracces
else:
reOpen = re.compile('{{2,}|\[{2,}')
reNext = re.compile('{{2,}|}{2,}|\[{2,}|]{2,}') # at least 2
reOpen = re.compile(r'{{2,}|\[{2,}')
reNext = re.compile(r'{{2,}|}{2,}|\[{2,}|]{2,}') # at least 2

cur = 0
while True:
Expand Down Expand Up @@ -1649,7 +1651,7 @@ def sharp_ifeq(lvalue, rvalue, valueIfTrue, valueIfFalse=None, *args):


def sharp_iferror(test, then='', Else=None, *args):
if re.match('<(?:strong|span|p|div)\s(?:[^\s>]*\s+)*?class="(?:[^"\s>]*\s+)*?error(?:\s[^">]*)?"', test):
if re.match(r'<(?:strong|span|p|div)\s(?:[^\s>]*\s+)*?class="(?:[^"\s>]*\s+)*?error(?:\s[^">]*)?"', test):
return then
elif Else is None:
return test.strip()
Expand Down Expand Up @@ -1820,7 +1822,7 @@ def define_template(title, page):
# title = normalizeTitle(title)

# check for redirects
m = re.match('#REDIRECT.*?\[\[([^\]]*)]]', page[0], re.IGNORECASE)
m = re.match(r'#REDIRECT.*?\[\[([^\]]*)]]', page[0], re.IGNORECASE)
if m:
redirects[title] = m.group(1) # normalizeTitle(m.group(1))
return
Expand Down