From aa479a7eed8e45cb13f021bd1537717494266a82 Mon Sep 17 00:00:00 2001 From: Mo Kweon Date: Tue, 11 Aug 2026 17:06:58 -0700 Subject: [PATCH 1/2] fix: derive POS tags from upstream semantics and restore -R variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generated postype.go did not compile: POS_P, POS_PV and POS_PA were all emitted as "P", which produced duplicate cases in the isValid switch. Rewrite the generator to evaluate POSTag enum initializers the way the C++ compiler does and to transcribe tagToString/tagRToString, instead of relying on hardcoded special cases and an "ends with i" heuristic. That corrects three mappings that never matched upstream: - pa = p + 1 == max, so tagToString returns "@", not "P" - pvi/pai fall through the irregular switch default, so both are "@", not "PV-I"/"PA-I" - the -R variants are reachable only through tagRToString, which is what kiwi_res_tag calls; dropping them made ParsePOSType fail on ordinary sentences such as "편지를 받았다", re-introducing #39 Deduplicate the isValid switch by value so aliases can no longer break the build, keep POS_USER_0..4 as deprecated aliases so the rename does not break downstream code, and add a regression test covering both the -R tags and a regular conjugation through Analyze. Verify the generated code inside the sync workflow: pull requests opened with GITHUB_TOKEN do not trigger CI, so nothing was checking it. --- .github/workflows/sync-postypes.yaml | 15 +- .gitignore | 3 + Makefile | 17 +- postype.go | 28 ++- postype_test.go | 49 +++++ scripts/extract_postags.py | 277 ++++++++++++++++++--------- 6 files changed, 289 insertions(+), 100 deletions(-) diff --git a/.github/workflows/sync-postypes.yaml b/.github/workflows/sync-postypes.yaml index f003a14..e5b6e19 100644 --- a/.github/workflows/sync-postypes.yaml +++ b/.github/workflows/sync-postypes.yaml @@ -41,13 +41,13 @@ jobs: id: get_version run: | if [ -n "${{ github.event.inputs.kiwi_version }}" ]; then - echo "version=${{ github.event.inputs.kiwi_version }}" >> $GITHUB_OUTPUT + VERSION="${{ github.event.inputs.kiwi_version }}" else # Get latest release from GitHub API VERSION=$(curl -s https://api.github.com/repos/bab2min/Kiwi/releases/latest | jq -r '.tag_name') - echo "version=$VERSION" >> $GITHUB_OUTPUT fi - echo "Kiwi version: $(cat $GITHUB_OUTPUT | grep version | cut -d= -f2)" + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Kiwi version: $VERSION" - name: Extract POS tags id: extract @@ -89,6 +89,15 @@ jobs: # Update Makefile version sed -i "s/KIWI_VERSION := .*/KIWI_VERSION := ${{ steps.get_version.outputs.version }}/" Makefile + # PRs opened with GITHUB_TOKEN do not trigger CI, so the generated code is + # verified here instead. Without this a non-compiling postype.go can be + # proposed and merged unnoticed. + - name: Verify generated code + if: steps.extract.outputs.changed == 'true' + run: | + make install-kiwi + make test + - name: Create Pull Request if: steps.extract.outputs.changed == 'true' uses: peter-evans/create-pull-request@v7 diff --git a/.gitignore b/.gitignore index 651c212..99ee9d1 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ ModelGenerator/ # Python virtual environment .venv/ + +# Generated by scripts/extract_postags.py before it replaces postype.go +postype_generated.go diff --git a/Makefile b/Makefile index 9be4093..96d0851 100644 --- a/Makefile +++ b/Makefile @@ -25,18 +25,27 @@ format: # go install mvdan.cc/gofumpt@latest gofumpt -l -w . +VENV := .venv +VENV_PYTHON := $(VENV)/bin/python + +# `source` is not available under /bin/sh on Debian-family systems, so the +# venv interpreter is invoked directly instead of activating the venv. +$(VENV_PYTHON): + python3 -m venv $(VENV) + $(VENV)/bin/pip install --quiet tree-sitter tree-sitter-cpp + .PHONY: sync-postypes -sync-postypes: +sync-postypes: $(VENV_PYTHON) @echo "Extracting POS tags from Kiwi $(KIWI_VERSION)..." - source .venv/bin/activate && python scripts/extract_postags.py $(KIWI_VERSION) + $(VENV_PYTHON) scripts/extract_postags.py $(KIWI_VERSION) @echo "Generated postype_generated.go" @echo "Comparing with current postype.go..." @diff -u postype.go postype_generated.go || true @echo "To apply changes, run: mv postype_generated.go postype.go" .PHONY: check-postypes -check-postypes: - @source .venv/bin/activate && python scripts/extract_postags.py $(KIWI_VERSION) +check-postypes: $(VENV_PYTHON) + @$(VENV_PYTHON) scripts/extract_postags.py $(KIWI_VERSION) @if diff -q postype.go postype_generated.go > /dev/null 2>&1; then \ echo "POS types are in sync with Kiwi $(KIWI_VERSION)"; \ else \ diff --git a/postype.go b/postype.go index 33bc154..9a43bcf 100644 --- a/postype.go +++ b/postype.go @@ -72,13 +72,26 @@ const ( POS_USER4 POSType = "USER4" POS_P POSType = "P" POS_PV POSType = "P" - POS_PA POSType = "P" + POS_PA POSType = "@" POS_VV_I POSType = "VV-I" POS_VA_I POSType = "VA-I" POS_VX_I POSType = "VX-I" POS_XSA_I POSType = "XSA-I" - POS_PV_I POSType = "PV-I" - POS_PA_I POSType = "PA-I" + POS_PV_I POSType = "@" + POS_PA_I POSType = "@" + POS_VV_R POSType = "VV-R" + POS_VA_R POSType = "VA-R" + POS_VX_R POSType = "VX-R" + POS_XSA_R POSType = "XSA-R" +) + +// Deprecated: these aliases are kept for backwards compatibility. +const ( + POS_USER_0 = POS_USER0 + POS_USER_1 = POS_USER1 + POS_USER_2 = POS_USER2 + POS_USER_3 = POS_USER3 + POS_USER_4 = POS_USER4 ) func (p POSType) isValid() bool { @@ -145,14 +158,15 @@ func (p POSType) isValid() bool { POS_USER3, POS_USER4, POS_P, - POS_PV, POS_PA, POS_VV_I, POS_VA_I, POS_VX_I, POS_XSA_I, - POS_PV_I, - POS_PA_I: + POS_VV_R, + POS_VA_R, + POS_VX_R, + POS_XSA_R: return true default: return false @@ -166,4 +180,4 @@ func ParsePOSType(t string) (POSType, error) { return POS_UNKNOWN, fmt.Errorf("POS type parse err. input type: %s", t) } return pos, nil -} \ No newline at end of file +} diff --git a/postype_test.go b/postype_test.go index 082d434..bd5287c 100644 --- a/postype_test.go +++ b/postype_test.go @@ -25,6 +25,32 @@ func TestParsePOSType(t *testing.T) { want: POS_UNKNOWN, wantErr: true, }, + // kiwi_res_tag is implemented with tagRToString, so the -R variants + // reach Go for regular conjugations. See #39. + { + name: "VV-R is a POSType", + arg: "VV-R", + want: POS_VV_R, + wantErr: false, + }, + { + name: "XSA-R is a POSType", + arg: "XSA-R", + want: POS_XSA_R, + wantErr: false, + }, + { + name: "VV-I is a POSType", + arg: "VV-I", + want: POS_VV_I, + wantErr: false, + }, + { + name: "@ is a POSType", + arg: "@", + want: POS_PA, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -39,3 +65,26 @@ func TestParsePOSType(t *testing.T) { }) } } + +// TestAnalyzeRegularConjugation guards against #39. Kiwi tags a regular verb +// whose stem ends in ㄷ/ㅂ/ㅅ as VV-R rather than VV, so dropping the -R +// constants makes Analyze fail on ordinary sentences. +func TestAnalyzeRegularConjugation(t *testing.T) { + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) + + for _, sentence := range []string{"편지를 받았다", "나는 공을 잡았다", "그는 크게 웃었다"} { + t.Run(sentence, func(t *testing.T) { + res, err := kiwi.Analyze(sentence) + assert.NoError(t, err) + + var tags []POSType + for _, result := range res { + for _, token := range result.Tokens { + tags = append(tags, token.Tag) + } + } + assert.Contains(t, tags, POS_VV_R) + }) + } +} diff --git a/scripts/extract_postags.py b/scripts/extract_postags.py index 3e91be2..e772a00 100644 --- a/scripts/extract_postags.py +++ b/scripts/extract_postags.py @@ -2,10 +2,16 @@ """ Extract POS tags from Kiwi C++ source code using tree-sitter. -This script parses the Kiwi C++ header file (Types.h) to extract -POSTag enum definitions and generates a Go file with the corresponding +This script parses the Kiwi C++ sources to extract the POSTag enum and the +tag<->string conversion tables, then generates a Go file with the corresponding POS type constants. +The mapping is derived rather than guessed: enum values are evaluated the way +the C++ compiler evaluates them (including aliases such as `pv = p` and +`pa = p + 1`), and the resulting numeric value is fed through a Python +transcription of `tagToString`. Tags that only `tagRToString` can produce +(the regular-conjugation `-R` variants) are parsed from that function. + Usage: python scripts/extract_postags.py [version] @@ -13,6 +19,7 @@ python scripts/extract_postags.py v0.23.2 """ +import re import sys import urllib.request from pathlib import Path @@ -27,6 +34,26 @@ KIWI_TYPES_H_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/include/kiwi/Types.h" KIWI_UTILS_CPP_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/src/Utils.cpp" +# Enumerators that are not POS tags themselves. +SKIPPED_ENUMERATORS = { + "max", # size marker + "irregular", # bit flag + "unknown_feat_ha", # internal use +} + +# Bases whose `i` spelling maps to a POS__I constant. +IRREGULAR_BASES = ("vv", "va", "vx", "xsa", "pv", "pa") + +# Backwards-compatible aliases kept for downstream code. These cannot be derived +# from the C++ source; they exist because kiwigo used to spell them this way. +COMPAT_ALIASES = [ + ("POS_USER_0", "POS_USER0"), + ("POS_USER_1", "POS_USER1"), + ("POS_USER_2", "POS_USER2"), + ("POS_USER_3", "POS_USER3"), + ("POS_USER_4", "POS_USER4"), +] + def fetch_file(url: str) -> str: """Fetch file content from URL.""" @@ -66,62 +93,135 @@ def traverse(node): return results -def extract_tag_strings(source: str) -> list[str]: - """Extract tag string array from Utils.cpp using regex.""" - import re +def resolve_enum_values(tags: list[dict]) -> dict[str, int]: + """Evaluate C++ enumerator initializers to their numeric values. + + Follows C++ rules: an enumerator without an initializer is the previous + value plus one, and an initializer may reference earlier enumerators. + """ + resolved: dict[str, int] = {} + previous = -1 + + for tag in tags: + expr = tag["value"] + if expr is None: + value = previous + 1 + else: + # Initializers in POSTag only use earlier enumerators, integer + # literals and the `+` / `|` operators. + if not re.fullmatch(r"[\w\s+|()x0-9A-Fa-f]+", expr): + raise ValueError(f"unsupported enumerator initializer: {expr!r}") + value = eval(expr, {"__builtins__": {}}, dict(resolved)) # noqa: S307 - # Find the tags array in tagToString function - # Pattern: static const char* tags[] = { "TAG1", "TAG2", ... } - pattern = r'static\s+const\s+char\s*\*\s*tags\s*\[\]\s*=\s*\{([^}]+)\}' - match = re.search(pattern, source, re.DOTALL) + resolved[tag["name"]] = value + previous = value + + return resolved + + +def extract_function_body(source: str, signature: str) -> str: + """Return the brace-delimited body of the first function matching `signature`.""" + match = re.search(signature, source) + if not match: + return "" + + start = source.find("{", match.end()) + if start == -1: + return "" + + depth = 0 + for i in range(start, len(source)): + if source[i] == "{": + depth += 1 + elif source[i] == "}": + depth -= 1 + if depth == 0: + return source[start : i + 1] + return "" + + +def extract_tag_strings(body: str) -> list[str]: + """Extract the `tags[]` string table from a tagToString-like function body.""" + pattern = r"static\s+const\s+char\s*\*\s*tags\s*\[\]\s*=\s*\{([^}]+)\}" + match = re.search(pattern, body, re.DOTALL) if not match: return [] - # Extract strings from the array - content = match.group(1) - string_pattern = r'"([^"]+)"' - return re.findall(string_pattern, content) + return re.findall(r'"([^"]+)"', match.group(1)) -def map_cpp_to_go_tag(cpp_name: str, tag_string: str) -> tuple[str, str]: - """Map C++ enum name to Go constant name and string value.""" - # Special cases - special_mappings = { - "unknown": ("POS_UNKNOWN", "UN"), - "irregular": None, # Skip - this is a flag, not a POS tag - "max": None, # Skip - this is a size marker - "p": ("POS_P", "P"), - "pv": ("POS_PV", "P"), # alias for p - "pa": ("POS_PA", "P"), # p + 1 - "unknown_feat_ha": None, # Skip - internal use - } +def extract_case_returns(body: str) -> dict[str, str]: + """Extract `case POSTag::x: return "Y";` pairs from a function body.""" + pattern = r'case\s+POSTag::(\w+)\s*:\s*return\s+"([^"]*)"' + return dict(re.findall(pattern, body)) - if cpp_name in special_mappings: - return special_mappings[cpp_name] - # Handle irregular variants (vvi, vai, vxi, xsai, pvi, pai) - if cpp_name.endswith("i") and len(cpp_name) > 2: - base = cpp_name[:-1] - if base in ("vv", "va", "vx", "xsa", "pv", "pa"): - go_name = f"POS_{base.upper()}_I" - go_value = f"{base.upper()}-I" - return (go_name, go_value) +def make_tag_to_string(tag_strings: list[str], irregular_cases: dict[str, str], + values: dict[str, int], irregular_flag: int): + """Build a Python transcription of the C++ `tagToString`.""" + # Map the numeric value of each irregular case label to its return string. + irregular_by_value = {values[name]: text for name, text in irregular_cases.items()} + default_irregular = "@" - # Default mapping: uppercase and add POS_ prefix - go_name = f"POS_{cpp_name.upper()}" - go_value = tag_string if tag_string else cpp_name.upper() + def tag_to_string(value: int) -> str | None: + if value & irregular_flag: + cleared = value & ~irregular_flag + return irregular_by_value.get(cleared, default_irregular) + if value >= len(tag_strings): + return None + return tag_strings[value] - return (go_name, go_value) + return tag_to_string -def generate_go_file(tags: list[dict], tag_strings: list[str], version: str) -> str: - """Generate Go source file with POS type definitions.""" - # Build tag string lookup - string_lookup = {} - for i, s in enumerate(tag_strings): - string_lookup[i] = s +def go_constant_name(cpp_name: str) -> str: + """Map a C++ enumerator name to its Go constant name.""" + if cpp_name == "unknown": + return "POS_UNKNOWN" + + if cpp_name.endswith("i") and cpp_name[:-1] in IRREGULAR_BASES: + return f"POS_{cpp_name[:-1].upper()}_I" + + return f"POS_{cpp_name.upper()}" + + +def build_tag_entries(tags: list[dict], values: dict[str, int], tag_to_string, + regular_cases: dict[str, str]) -> list[tuple[str, str]]: + """Build the ordered (go_name, go_value) list for the generated constants.""" + entries: list[tuple[str, str]] = [] + seen_names: set[str] = set() + + for tag in tags: + cpp_name = tag["name"] + if cpp_name in SKIPPED_ENUMERATORS: + continue + + go_value = tag_to_string(values[cpp_name]) + if go_value is None: + continue + + go_name = go_constant_name(cpp_name) + if go_name in seen_names: + continue + seen_names.add(go_name) + entries.append((go_name, go_value)) + + # `-R` variants are only reachable through tagRToString, so they are not + # derivable from the enum alone. + for cpp_name, text in regular_cases.items(): + go_name = f"POS_{cpp_name.upper()}_R" + if go_name in seen_names: + continue + seen_names.add(go_name) + entries.append((go_name, text)) + + return entries + + +def generate_go_file(entries: list[tuple[str, str]], version: str) -> str: + """Generate Go source file with POS type definitions.""" lines = [] lines.append("// Code generated by scripts/extract_postags.py; DO NOT EDIT.") lines.append(f"// Source: Kiwi {version}") @@ -136,50 +236,37 @@ def generate_go_file(tags: list[dict], tag_strings: list[str], version: str) -> lines.append("") lines.append("const (") - # Track which tags we've added and collect for alignment - added_tags = set() - tag_entries = [] - - for i, tag in enumerate(tags): - cpp_name = tag["name"] - tag_string = string_lookup.get(i, cpp_name.upper()) + max_name_len = max(len(name) for name, _ in entries) - result = map_cpp_to_go_tag(cpp_name, tag_string) - if result is None: - continue - - go_name, go_value = result - if go_name in added_tags: - continue - - added_tags.add(go_name) - tag_entries.append((go_name, go_value)) - - # Calculate max name length for alignment - max_name_len = max(len(name) for name, _ in tag_entries) - - # Generate aligned output - for go_name, go_value in tag_entries: + for go_name, go_value in entries: padding = " " * (max_name_len - len(go_name) + 1) lines.append(f'\t{go_name}{padding}POSType = "{go_value}"') lines.append(")") lines.append("") - # Generate isValid function + # Backwards-compatible aliases. + lines.append("// Deprecated: these aliases are kept for backwards compatibility.") + lines.append("const (") + alias_len = max(len(old) for old, _ in COMPAT_ALIASES) + for old_name, new_name in COMPAT_ALIASES: + padding = " " * (alias_len - len(old_name) + 1) + lines.append(f"\t{old_name}{padding}= {new_name}") + lines.append(")") + lines.append("") + + # Generate isValid function. A Go switch requires distinct case values, and + # several constants are aliases sharing one string, so deduplicate by value. lines.append("func (p POSType) isValid() bool {") lines.append("\tswitch p {") lines.append("\tcase") valid_tags = [] - for tag in tags: - cpp_name = tag["name"] - result = map_cpp_to_go_tag(cpp_name, "") - if result is None: - continue - go_name = result[0] - if go_name not in added_tags: + seen_values: set[str] = set() + for go_name, go_value in entries: + if go_value in seen_values: continue + seen_values.add(go_value) valid_tags.append(go_name) lines.append(",\n".join(f"\t\t{tag}" for tag in valid_tags) + ":") @@ -200,7 +287,7 @@ def generate_go_file(tags: list[dict], tag_strings: list[str], version: str) -> lines.append("\treturn pos, nil") lines.append("}") - return "\n".join(lines) + return "\n".join(lines) + "\n" def main(): @@ -223,13 +310,29 @@ def main(): tags = extract_enum_values(types_source, "POSTag") print(f"Found {len(tags)} enum values") - # Extract tag strings + values = resolve_enum_values(tags) + if "irregular" not in values: + raise SystemExit("POSTag::irregular not found; cannot decode irregular tags") + + # Extract the conversion tables. print("Parsing tag strings...") - tag_strings = extract_tag_strings(utils_source) + tag_to_string_body = extract_function_body(utils_source, r"const\s+char\s*\*\s*tagToString\s*\(") + tag_strings = extract_tag_strings(tag_to_string_body) + if not tag_strings: + raise SystemExit("tagToString tag table not found") print(f"Found {len(tag_strings)} tag strings") + irregular_cases = extract_case_returns(tag_to_string_body) + + tag_r_to_string_body = extract_function_body(utils_source, r"const\s+char\s*\*\s*tagRToString\s*\(") + regular_cases = extract_case_returns(tag_r_to_string_body) + print(f"Found {len(regular_cases)} regular-conjugation tags") + + tag_to_string = make_tag_to_string(tag_strings, irregular_cases, values, values["irregular"]) + entries = build_tag_entries(tags, values, tag_to_string, regular_cases) + # Generate Go file - go_content = generate_go_file(tags, tag_strings, version) + go_content = generate_go_file(entries, version) # Write output output_path = Path("postype_generated.go") @@ -238,13 +341,15 @@ def main(): # Print summary print("\nExtracted tags:") - for i, tag in enumerate(tags): - tag_str = tag_strings[i] if i < len(tag_strings) else "?" - result = map_cpp_to_go_tag(tag["name"], tag_str) - if result: - print(f" {tag['name']:20s} -> {result[0]:20s} = \"{result[1]}\"") - else: - print(f" {tag['name']:20s} -> (skipped)") + for tag in tags: + cpp_name = tag["name"] + if cpp_name in SKIPPED_ENUMERATORS: + print(f" {cpp_name:20s} -> (skipped)") + continue + text = tag_to_string(values[cpp_name]) + print(f' {cpp_name:20s} -> {go_constant_name(cpp_name):20s} = "{text}"') + for cpp_name, text in regular_cases.items(): + print(f' {cpp_name + " (R)":20s} -> {"POS_" + cpp_name.upper() + "_R":20s} = "{text}"') if __name__ == "__main__": From 6c5f2893b3b983f2b08512664057db6b7b561f3b Mon Sep 17 00:00:00 2001 From: Mo Kweon Date: Tue, 11 Aug 2026 17:29:01 -0700 Subject: [PATCH 2/2] fix: mirror upstream toPOSTag for the accepted tag set ParsePOSType is the Go counterpart of upstream's toPOSTag, so derive the accepted strings from that function instead of inventing them. The previous commit mapped POS_PA, POS_PV_I and POS_PA_I to "@". That string is not something upstream ever converts to or from: tagToString guards its table with assert(t < POSTag::max) and pa == max, so reading "@" only happens in an NDEBUG build past the assertion, and toPOSTag has no entry for "@" at all. Drop the three constants rather than exposing a sentinel as a tag. Restore POS_V and add the remaining strings toPOSTag accepts but Kiwi never emits: A (POSTag::p), NF, NV, NA and UNK (POSTag::unknown), and ^ (POSTag::unknown, spelled POS_CARET). POS_V had been removed on the grounds that Kiwi never returns "V", which is true of tagToString but irrelevant to a parser -- toPOSTag accepts it. kiwigo now accepts exactly toPOSTag's 75 strings plus "P", which tagToString can return for POSTag::p even though toPOSTag does not accept it. Emit the two groups as separate blocks so the generated alignment matches gofmt. --- postype.go | 22 ++++-- postype_test.go | 21 +++++- scripts/extract_postags.py | 133 ++++++++++++++++++++++++++++--------- 3 files changed, 135 insertions(+), 41 deletions(-) diff --git a/postype.go b/postype.go index 9a43bcf..97e4b1c 100644 --- a/postype.go +++ b/postype.go @@ -72,17 +72,23 @@ const ( POS_USER4 POSType = "USER4" POS_P POSType = "P" POS_PV POSType = "P" - POS_PA POSType = "@" POS_VV_I POSType = "VV-I" POS_VA_I POSType = "VA-I" POS_VX_I POSType = "VX-I" POS_XSA_I POSType = "XSA-I" - POS_PV_I POSType = "@" - POS_PA_I POSType = "@" POS_VV_R POSType = "VV-R" POS_VA_R POSType = "VA-R" POS_VX_R POSType = "VX-R" POS_XSA_R POSType = "XSA-R" + + // Accepted by toPOSTag but never produced by Kiwi itself. + POS_NF POSType = "NF" + POS_NV POSType = "NV" + POS_NA POSType = "NA" + POS_UNK POSType = "UNK" + POS_V POSType = "V" + POS_A POSType = "A" + POS_CARET POSType = "^" ) // Deprecated: these aliases are kept for backwards compatibility. @@ -158,7 +164,6 @@ func (p POSType) isValid() bool { POS_USER3, POS_USER4, POS_P, - POS_PA, POS_VV_I, POS_VA_I, POS_VX_I, @@ -166,7 +171,14 @@ func (p POSType) isValid() bool { POS_VV_R, POS_VA_R, POS_VX_R, - POS_XSA_R: + POS_XSA_R, + POS_NF, + POS_NV, + POS_NA, + POS_UNK, + POS_V, + POS_A, + POS_CARET: return true default: return false diff --git a/postype_test.go b/postype_test.go index bd5287c..6e7190a 100644 --- a/postype_test.go +++ b/postype_test.go @@ -45,12 +45,27 @@ func TestParsePOSType(t *testing.T) { want: POS_VV_I, wantErr: false, }, + // toPOSTag accepts these as input even though Kiwi never emits them. { - name: "@ is a POSType", - arg: "@", - want: POS_PA, + name: "V is a POSType", + arg: "V", + want: POS_V, wantErr: false, }, + { + name: "UNK is a POSType", + arg: "UNK", + want: POS_UNK, + wantErr: false, + }, + // toPOSTag has no entry for "@"; it is the sentinel tagToString falls + // back to for values it is never meant to be called with. + { + name: "@ is not a valid POSType", + arg: "@", + want: POS_UNKNOWN, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/scripts/extract_postags.py b/scripts/extract_postags.py index e772a00..99a6476 100644 --- a/scripts/extract_postags.py +++ b/scripts/extract_postags.py @@ -33,6 +33,7 @@ # GitHub raw content URL template KIWI_TYPES_H_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/include/kiwi/Types.h" KIWI_UTILS_CPP_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/src/Utils.cpp" +KIWI_STRUTILS_H_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/src/StrUtils.h" # Enumerators that are not POS tags themselves. SKIPPED_ENUMERATORS = { @@ -44,6 +45,12 @@ # Bases whose `i` spelling maps to a POS__I constant. IRREGULAR_BASES = ("vv", "va", "vx", "xsa", "pv", "pa") +# Go constant names for tag strings that toPOSTag accepts but that are not +# spelled like an identifier. +PUNCTUATION_TAG_NAMES = { + "^": "POS_CARET", +} + # Backwards-compatible aliases kept for downstream code. These cannot be derived # from the C++ source; they exist because kiwigo used to spell them this way. COMPAT_ALIASES = [ @@ -157,18 +164,37 @@ def extract_case_returns(body: str) -> dict[str, str]: return dict(re.findall(pattern, body)) +def extract_accepted_tags(body: str) -> list[str]: + """Extract the tag strings that `toPOSTag` accepts, in source order. + + toPOSTag is the upstream counterpart of ParsePOSType, so it defines which + strings are valid input. Anything it does not recognise falls through to + `return POSTag::max`, which is upstream's way of rejecting the string. + """ + pattern = r'tagStr\s*==\s*u"([^"]*)"\s*\)\s*return' + seen: list[str] = [] + for text in re.findall(pattern, body): + if text not in seen: + seen.append(text) + return seen + + def make_tag_to_string(tag_strings: list[str], irregular_cases: dict[str, str], - values: dict[str, int], irregular_flag: int): - """Build a Python transcription of the C++ `tagToString`.""" + values: dict[str, int], irregular_flag: int, max_value: int): + """Build a Python transcription of the C++ `tagToString`. + + Returns None where upstream has no tag string for the value. tagToString + guards its table lookup with `assert(t < POSTag::max)` and its irregular + branch falls through to a "@" sentinel, so those values are not tags that + upstream ever converts; `toPOSTag` rejects "@" as well. + """ # Map the numeric value of each irregular case label to its return string. irregular_by_value = {values[name]: text for name, text in irregular_cases.items()} - default_irregular = "@" def tag_to_string(value: int) -> str | None: if value & irregular_flag: - cleared = value & ~irregular_flag - return irregular_by_value.get(cleared, default_irregular) - if value >= len(tag_strings): + return irregular_by_value.get(value & ~irregular_flag) + if value >= max_value: return None return tag_strings[value] @@ -187,10 +213,23 @@ def go_constant_name(cpp_name: str) -> str: def build_tag_entries(tags: list[dict], values: dict[str, int], tag_to_string, - regular_cases: dict[str, str]) -> list[tuple[str, str]]: - """Build the ordered (go_name, go_value) list for the generated constants.""" - entries: list[tuple[str, str]] = [] + regular_cases: dict[str, str], + accepted: list[str]) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]: + """Build the (go_name, go_value) lists for the generated constants. + + Returns the tags Kiwi can emit and, separately, the input-only aliases that + upstream's toPOSTag accepts but never produces. + """ + emitted: list[tuple[str, str]] = [] seen_names: set[str] = set() + seen_values: set[str] = set() + + def add(go_name: str, go_value: str, target: list[tuple[str, str]]) -> None: + if go_name in seen_names: + return + seen_names.add(go_name) + seen_values.add(go_value) + target.append((go_name, go_value)) for tag in tags: cpp_name = tag["name"] @@ -201,27 +240,29 @@ def build_tag_entries(tags: list[dict], values: dict[str, int], tag_to_string, if go_value is None: continue - go_name = go_constant_name(cpp_name) - if go_name in seen_names: - continue - - seen_names.add(go_name) - entries.append((go_name, go_value)) + add(go_constant_name(cpp_name), go_value, emitted) # `-R` variants are only reachable through tagRToString, so they are not # derivable from the enum alone. for cpp_name, text in regular_cases.items(): - go_name = f"POS_{cpp_name.upper()}_R" - if go_name in seen_names: + add(f"POS_{cpp_name.upper()}_R", text, emitted) + + # Strings toPOSTag accepts as input but tagToString never returns, such as + # "V" and "A" for POSTag::p or "NF"/"NV"/"NA"/"UNK" for POSTag::unknown. + aliases: list[tuple[str, str]] = [] + for text in accepted: + if text in seen_values: continue - seen_names.add(go_name) - entries.append((go_name, text)) + go_name = PUNCTUATION_TAG_NAMES.get(text, f"POS_{text.upper()}") + add(go_name, text, aliases) - return entries + return emitted, aliases -def generate_go_file(entries: list[tuple[str, str]], version: str) -> str: +def generate_go_file(emitted: list[tuple[str, str]], aliases: list[tuple[str, str]], + version: str) -> str: """Generate Go source file with POS type definitions.""" + entries = emitted + aliases lines = [] lines.append("// Code generated by scripts/extract_postags.py; DO NOT EDIT.") lines.append(f"// Source: Kiwi {version}") @@ -236,11 +277,19 @@ def generate_go_file(entries: list[tuple[str, str]], version: str) -> str: lines.append("") lines.append("const (") - max_name_len = max(len(name) for name, _ in entries) + # gofmt aligns each run of declarations separated by a blank line on its + # own, so each group is padded to its own longest name. + def declare_group(group: list[tuple[str, str]]) -> list[str]: + width = max(len(name) for name, _ in group) + return [ + f'\t{go_name}{" " * (width - len(go_name) + 1)}POSType = "{go_value}"' + for go_name, go_value in group + ] - for go_name, go_value in entries: - padding = " " * (max_name_len - len(go_name) + 1) - lines.append(f'\t{go_name}{padding}POSType = "{go_value}"') + lines.extend(declare_group(emitted)) + lines.append("") + lines.append("\t// Accepted by toPOSTag but never produced by Kiwi itself.") + lines.extend(declare_group(aliases)) lines.append(")") lines.append("") @@ -255,8 +304,10 @@ def generate_go_file(entries: list[tuple[str, str]], version: str) -> str: lines.append(")") lines.append("") - # Generate isValid function. A Go switch requires distinct case values, and - # several constants are aliases sharing one string, so deduplicate by value. + # Generate isValid function. This mirrors which strings upstream's toPOSTag + # accepts, plus the strings tagToString/tagRToString can return. A Go switch + # requires distinct case values, and several constants are aliases sharing + # one string, so deduplicate by value. lines.append("func (p POSType) isValid() bool {") lines.append("\tswitch p {") lines.append("\tcase") @@ -305,6 +356,10 @@ def main(): print(f"Fetching {utils_url}") utils_source = fetch_file(utils_url) + strutils_url = KIWI_STRUTILS_H_URL.format(version=version) + print(f"Fetching {strutils_url}") + strutils_source = fetch_file(strutils_url) + # Extract enum values print("Parsing POSTag enum...") tags = extract_enum_values(types_source, "POSTag") @@ -328,11 +383,21 @@ def main(): regular_cases = extract_case_returns(tag_r_to_string_body) print(f"Found {len(regular_cases)} regular-conjugation tags") - tag_to_string = make_tag_to_string(tag_strings, irregular_cases, values, values["irregular"]) - entries = build_tag_entries(tags, values, tag_to_string, regular_cases) + # toPOSTag defines which strings upstream accepts as input, which is the + # role ParsePOSType plays on the Go side. + to_pos_tag_body = extract_function_body(strutils_source, r"inline\s+POSTag\s+toPOSTag\s*\(\s*std::u16string_view") + accepted = extract_accepted_tags(to_pos_tag_body) + if not accepted: + raise SystemExit("toPOSTag not found; cannot determine accepted tag strings") + print(f"Found {len(accepted)} accepted tag strings") + + tag_to_string = make_tag_to_string( + tag_strings, irregular_cases, values, values["irregular"], values["max"] + ) + emitted, aliases = build_tag_entries(tags, values, tag_to_string, regular_cases, accepted) # Generate Go file - go_content = generate_go_file(entries, version) + go_content = generate_go_file(emitted, aliases, version) # Write output output_path = Path("postype_generated.go") @@ -343,13 +408,15 @@ def main(): print("\nExtracted tags:") for tag in tags: cpp_name = tag["name"] - if cpp_name in SKIPPED_ENUMERATORS: - print(f" {cpp_name:20s} -> (skipped)") + text = None if cpp_name in SKIPPED_ENUMERATORS else tag_to_string(values[cpp_name]) + if text is None: + print(f" {cpp_name:20s} -> (no tag string; skipped)") continue - text = tag_to_string(values[cpp_name]) print(f' {cpp_name:20s} -> {go_constant_name(cpp_name):20s} = "{text}"') for cpp_name, text in regular_cases.items(): print(f' {cpp_name + " (R)":20s} -> {"POS_" + cpp_name.upper() + "_R":20s} = "{text}"') + for go_name, text in aliases: + print(f' {"(toPOSTag only)":20s} -> {go_name:20s} = "{text}"') if __name__ == "__main__":