diff --git a/.github/workflows/sync-postypes.yaml b/.github/workflows/sync-postypes.yaml
index f003a14..e5b6e19 100644
--- a/.github/workflows/sync-postypes.yaml
+++ b/.github/workflows/sync-postypes.yaml
@@ -41,13 +41,13 @@ jobs:
id: get_version
run: |
if [ -n "${{ github.event.inputs.kiwi_version }}" ]; then
- echo "version=${{ github.event.inputs.kiwi_version }}" >> $GITHUB_OUTPUT
+ VERSION="${{ github.event.inputs.kiwi_version }}"
else
# Get latest release from GitHub API
VERSION=$(curl -s https://api.github.com/repos/bab2min/Kiwi/releases/latest | jq -r '.tag_name')
- echo "version=$VERSION" >> $GITHUB_OUTPUT
fi
- echo "Kiwi version: $(cat $GITHUB_OUTPUT | grep version | cut -d= -f2)"
+ echo "version=$VERSION" >> $GITHUB_OUTPUT
+ echo "Kiwi version: $VERSION"
- name: Extract POS tags
id: extract
@@ -89,6 +89,15 @@ jobs:
# Update Makefile version
sed -i "s/KIWI_VERSION := .*/KIWI_VERSION := ${{ steps.get_version.outputs.version }}/" Makefile
+ # PRs opened with GITHUB_TOKEN do not trigger CI, so the generated code is
+ # verified here instead. Without this a non-compiling postype.go can be
+ # proposed and merged unnoticed.
+ - name: Verify generated code
+ if: steps.extract.outputs.changed == 'true'
+ run: |
+ make install-kiwi
+ make test
+
- name: Create Pull Request
if: steps.extract.outputs.changed == 'true'
uses: peter-evans/create-pull-request@v7
diff --git a/.gitignore b/.gitignore
index 651c212..99ee9d1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,6 @@ ModelGenerator/
# Python virtual environment
.venv/
+
+# Generated by scripts/extract_postags.py before it replaces postype.go
+postype_generated.go
diff --git a/Makefile b/Makefile
index 9be4093..96d0851 100644
--- a/Makefile
+++ b/Makefile
@@ -25,18 +25,27 @@ format:
# go install mvdan.cc/gofumpt@latest
gofumpt -l -w .
+VENV := .venv
+VENV_PYTHON := $(VENV)/bin/python
+
+# `source` is not available under /bin/sh on Debian-family systems, so the
+# venv interpreter is invoked directly instead of activating the venv.
+$(VENV_PYTHON):
+ python3 -m venv $(VENV)
+ $(VENV)/bin/pip install --quiet tree-sitter tree-sitter-cpp
+
.PHONY: sync-postypes
-sync-postypes:
+sync-postypes: $(VENV_PYTHON)
@echo "Extracting POS tags from Kiwi $(KIWI_VERSION)..."
- source .venv/bin/activate && python scripts/extract_postags.py $(KIWI_VERSION)
+ $(VENV_PYTHON) scripts/extract_postags.py $(KIWI_VERSION)
@echo "Generated postype_generated.go"
@echo "Comparing with current postype.go..."
@diff -u postype.go postype_generated.go || true
@echo "To apply changes, run: mv postype_generated.go postype.go"
.PHONY: check-postypes
-check-postypes:
- @source .venv/bin/activate && python scripts/extract_postags.py $(KIWI_VERSION)
+check-postypes: $(VENV_PYTHON)
+ @$(VENV_PYTHON) scripts/extract_postags.py $(KIWI_VERSION)
@if diff -q postype.go postype_generated.go > /dev/null 2>&1; then \
echo "POS types are in sync with Kiwi $(KIWI_VERSION)"; \
else \
diff --git a/postype.go b/postype.go
index 33bc154..97e4b1c 100644
--- a/postype.go
+++ b/postype.go
@@ -72,13 +72,32 @@ const (
POS_USER4 POSType = "USER4"
POS_P POSType = "P"
POS_PV POSType = "P"
- POS_PA POSType = "P"
POS_VV_I POSType = "VV-I"
POS_VA_I POSType = "VA-I"
POS_VX_I POSType = "VX-I"
POS_XSA_I POSType = "XSA-I"
- POS_PV_I POSType = "PV-I"
- POS_PA_I POSType = "PA-I"
+ POS_VV_R POSType = "VV-R"
+ POS_VA_R POSType = "VA-R"
+ POS_VX_R POSType = "VX-R"
+ POS_XSA_R POSType = "XSA-R"
+
+ // Accepted by toPOSTag but never produced by Kiwi itself.
+ POS_NF POSType = "NF"
+ POS_NV POSType = "NV"
+ POS_NA POSType = "NA"
+ POS_UNK POSType = "UNK"
+ POS_V POSType = "V"
+ POS_A POSType = "A"
+ POS_CARET POSType = "^"
+)
+
+// Deprecated: these aliases are kept for backwards compatibility.
+const (
+ POS_USER_0 = POS_USER0
+ POS_USER_1 = POS_USER1
+ POS_USER_2 = POS_USER2
+ POS_USER_3 = POS_USER3
+ POS_USER_4 = POS_USER4
)
func (p POSType) isValid() bool {
@@ -145,14 +164,21 @@ func (p POSType) isValid() bool {
POS_USER3,
POS_USER4,
POS_P,
- POS_PV,
- POS_PA,
POS_VV_I,
POS_VA_I,
POS_VX_I,
POS_XSA_I,
- POS_PV_I,
- POS_PA_I:
+ POS_VV_R,
+ POS_VA_R,
+ POS_VX_R,
+ POS_XSA_R,
+ POS_NF,
+ POS_NV,
+ POS_NA,
+ POS_UNK,
+ POS_V,
+ POS_A,
+ POS_CARET:
return true
default:
return false
@@ -166,4 +192,4 @@ func ParsePOSType(t string) (POSType, error) {
return POS_UNKNOWN, fmt.Errorf("POS type parse err. input type: %s", t)
}
return pos, nil
-}
\ No newline at end of file
+}
diff --git a/postype_test.go b/postype_test.go
index 082d434..6e7190a 100644
--- a/postype_test.go
+++ b/postype_test.go
@@ -25,6 +25,47 @@ func TestParsePOSType(t *testing.T) {
want: POS_UNKNOWN,
wantErr: true,
},
+ // kiwi_res_tag is implemented with tagRToString, so the -R variants
+ // reach Go for regular conjugations. See #39.
+ {
+ name: "VV-R is a POSType",
+ arg: "VV-R",
+ want: POS_VV_R,
+ wantErr: false,
+ },
+ {
+ name: "XSA-R is a POSType",
+ arg: "XSA-R",
+ want: POS_XSA_R,
+ wantErr: false,
+ },
+ {
+ name: "VV-I is a POSType",
+ arg: "VV-I",
+ want: POS_VV_I,
+ wantErr: false,
+ },
+ // toPOSTag accepts these as input even though Kiwi never emits them.
+ {
+ name: "V is a POSType",
+ arg: "V",
+ want: POS_V,
+ wantErr: false,
+ },
+ {
+ name: "UNK is a POSType",
+ arg: "UNK",
+ want: POS_UNK,
+ wantErr: false,
+ },
+ // toPOSTag has no entry for "@"; it is the sentinel tagToString falls
+ // back to for values it is never meant to be called with.
+ {
+ name: "@ is not a valid POSType",
+ arg: "@",
+ want: POS_UNKNOWN,
+ wantErr: true,
+ },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -39,3 +80,26 @@ func TestParsePOSType(t *testing.T) {
})
}
}
+
+// TestAnalyzeRegularConjugation guards against #39. Kiwi tags a regular verb
+// whose stem ends in ㄷ/ㅂ/ㅅ as VV-R rather than VV, so dropping the -R
+// constants makes Analyze fail on ordinary sentences.
+func TestAnalyzeRegularConjugation(t *testing.T) {
+ kiwi, err := New("./base", WithNumThread(1))
+ assert.NoError(t, err)
+
+ for _, sentence := range []string{"편지를 받았다", "나는 공을 잡았다", "그는 크게 웃었다"} {
+ t.Run(sentence, func(t *testing.T) {
+ res, err := kiwi.Analyze(sentence)
+ assert.NoError(t, err)
+
+ var tags []POSType
+ for _, result := range res {
+ for _, token := range result.Tokens {
+ tags = append(tags, token.Tag)
+ }
+ }
+ assert.Contains(t, tags, POS_VV_R)
+ })
+ }
+}
diff --git a/scripts/extract_postags.py b/scripts/extract_postags.py
index 3e91be2..99a6476 100644
--- a/scripts/extract_postags.py
+++ b/scripts/extract_postags.py
@@ -2,10 +2,16 @@
"""
Extract POS tags from Kiwi C++ source code using tree-sitter.
-This script parses the Kiwi C++ header file (Types.h) to extract
-POSTag enum definitions and generates a Go file with the corresponding
+This script parses the Kiwi C++ sources to extract the POSTag enum and the
+tag<->string conversion tables, then generates a Go file with the corresponding
POS type constants.
+The mapping is derived rather than guessed: enum values are evaluated the way
+the C++ compiler evaluates them (including aliases such as `pv = p` and
+`pa = p + 1`), and the resulting numeric value is fed through a Python
+transcription of `tagToString`. Tags that only `tagRToString` can produce
+(the regular-conjugation `-R` variants) are parsed from that function.
+
Usage:
python scripts/extract_postags.py [version]
@@ -13,6 +19,7 @@
python scripts/extract_postags.py v0.23.2
"""
+import re
import sys
import urllib.request
from pathlib import Path
@@ -26,6 +33,33 @@
# GitHub raw content URL template
KIWI_TYPES_H_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/include/kiwi/Types.h"
KIWI_UTILS_CPP_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/src/Utils.cpp"
+KIWI_STRUTILS_H_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/src/StrUtils.h"
+
+# Enumerators that are not POS tags themselves.
+SKIPPED_ENUMERATORS = {
+ "max", # size marker
+ "irregular", # bit flag
+ "unknown_feat_ha", # internal use
+}
+
+# Bases whose `i` spelling maps to a POS__I constant.
+IRREGULAR_BASES = ("vv", "va", "vx", "xsa", "pv", "pa")
+
+# Go constant names for tag strings that toPOSTag accepts but that are not
+# spelled like an identifier.
+PUNCTUATION_TAG_NAMES = {
+ "^": "POS_CARET",
+}
+
+# Backwards-compatible aliases kept for downstream code. These cannot be derived
+# from the C++ source; they exist because kiwigo used to spell them this way.
+COMPAT_ALIASES = [
+ ("POS_USER_0", "POS_USER0"),
+ ("POS_USER_1", "POS_USER1"),
+ ("POS_USER_2", "POS_USER2"),
+ ("POS_USER_3", "POS_USER3"),
+ ("POS_USER_4", "POS_USER4"),
+]
def fetch_file(url: str) -> str:
@@ -66,62 +100,169 @@ def traverse(node):
return results
-def extract_tag_strings(source: str) -> list[str]:
- """Extract tag string array from Utils.cpp using regex."""
- import re
+def resolve_enum_values(tags: list[dict]) -> dict[str, int]:
+ """Evaluate C++ enumerator initializers to their numeric values.
+
+ Follows C++ rules: an enumerator without an initializer is the previous
+ value plus one, and an initializer may reference earlier enumerators.
+ """
+ resolved: dict[str, int] = {}
+ previous = -1
+
+ for tag in tags:
+ expr = tag["value"]
+ if expr is None:
+ value = previous + 1
+ else:
+ # Initializers in POSTag only use earlier enumerators, integer
+ # literals and the `+` / `|` operators.
+ if not re.fullmatch(r"[\w\s+|()x0-9A-Fa-f]+", expr):
+ raise ValueError(f"unsupported enumerator initializer: {expr!r}")
+ value = eval(expr, {"__builtins__": {}}, dict(resolved)) # noqa: S307
+
+ resolved[tag["name"]] = value
+ previous = value
+
+ return resolved
+
+
+def extract_function_body(source: str, signature: str) -> str:
+ """Return the brace-delimited body of the first function matching `signature`."""
+ match = re.search(signature, source)
+ if not match:
+ return ""
+
+ start = source.find("{", match.end())
+ if start == -1:
+ return ""
+
+ depth = 0
+ for i in range(start, len(source)):
+ if source[i] == "{":
+ depth += 1
+ elif source[i] == "}":
+ depth -= 1
+ if depth == 0:
+ return source[start : i + 1]
+ return ""
- # Find the tags array in tagToString function
- # Pattern: static const char* tags[] = { "TAG1", "TAG2", ... }
- pattern = r'static\s+const\s+char\s*\*\s*tags\s*\[\]\s*=\s*\{([^}]+)\}'
- match = re.search(pattern, source, re.DOTALL)
+
+def extract_tag_strings(body: str) -> list[str]:
+ """Extract the `tags[]` string table from a tagToString-like function body."""
+ pattern = r"static\s+const\s+char\s*\*\s*tags\s*\[\]\s*=\s*\{([^}]+)\}"
+ match = re.search(pattern, body, re.DOTALL)
if not match:
return []
- # Extract strings from the array
- content = match.group(1)
- string_pattern = r'"([^"]+)"'
- return re.findall(string_pattern, content)
+ return re.findall(r'"([^"]+)"', match.group(1))
-def map_cpp_to_go_tag(cpp_name: str, tag_string: str) -> tuple[str, str]:
- """Map C++ enum name to Go constant name and string value."""
- # Special cases
- special_mappings = {
- "unknown": ("POS_UNKNOWN", "UN"),
- "irregular": None, # Skip - this is a flag, not a POS tag
- "max": None, # Skip - this is a size marker
- "p": ("POS_P", "P"),
- "pv": ("POS_PV", "P"), # alias for p
- "pa": ("POS_PA", "P"), # p + 1
- "unknown_feat_ha": None, # Skip - internal use
- }
+def extract_case_returns(body: str) -> dict[str, str]:
+ """Extract `case POSTag::x: return "Y";` pairs from a function body."""
+ pattern = r'case\s+POSTag::(\w+)\s*:\s*return\s+"([^"]*)"'
+ return dict(re.findall(pattern, body))
- if cpp_name in special_mappings:
- return special_mappings[cpp_name]
- # Handle irregular variants (vvi, vai, vxi, xsai, pvi, pai)
- if cpp_name.endswith("i") and len(cpp_name) > 2:
- base = cpp_name[:-1]
- if base in ("vv", "va", "vx", "xsa", "pv", "pa"):
- go_name = f"POS_{base.upper()}_I"
- go_value = f"{base.upper()}-I"
- return (go_name, go_value)
+def extract_accepted_tags(body: str) -> list[str]:
+ """Extract the tag strings that `toPOSTag` accepts, in source order.
- # Default mapping: uppercase and add POS_ prefix
- go_name = f"POS_{cpp_name.upper()}"
- go_value = tag_string if tag_string else cpp_name.upper()
+ toPOSTag is the upstream counterpart of ParsePOSType, so it defines which
+ strings are valid input. Anything it does not recognise falls through to
+ `return POSTag::max`, which is upstream's way of rejecting the string.
+ """
+ pattern = r'tagStr\s*==\s*u"([^"]*)"\s*\)\s*return'
+ seen: list[str] = []
+ for text in re.findall(pattern, body):
+ if text not in seen:
+ seen.append(text)
+ return seen
- return (go_name, go_value)
+def make_tag_to_string(tag_strings: list[str], irregular_cases: dict[str, str],
+ values: dict[str, int], irregular_flag: int, max_value: int):
+ """Build a Python transcription of the C++ `tagToString`.
-def generate_go_file(tags: list[dict], tag_strings: list[str], version: str) -> str:
- """Generate Go source file with POS type definitions."""
- # Build tag string lookup
- string_lookup = {}
- for i, s in enumerate(tag_strings):
- string_lookup[i] = s
+ Returns None where upstream has no tag string for the value. tagToString
+ guards its table lookup with `assert(t < POSTag::max)` and its irregular
+ branch falls through to a "@" sentinel, so those values are not tags that
+ upstream ever converts; `toPOSTag` rejects "@" as well.
+ """
+ # Map the numeric value of each irregular case label to its return string.
+ irregular_by_value = {values[name]: text for name, text in irregular_cases.items()}
+
+ def tag_to_string(value: int) -> str | None:
+ if value & irregular_flag:
+ return irregular_by_value.get(value & ~irregular_flag)
+ if value >= max_value:
+ return None
+ return tag_strings[value]
+
+ return tag_to_string
+
+
+def go_constant_name(cpp_name: str) -> str:
+ """Map a C++ enumerator name to its Go constant name."""
+ if cpp_name == "unknown":
+ return "POS_UNKNOWN"
+
+ if cpp_name.endswith("i") and cpp_name[:-1] in IRREGULAR_BASES:
+ return f"POS_{cpp_name[:-1].upper()}_I"
+
+ return f"POS_{cpp_name.upper()}"
+
+
+def build_tag_entries(tags: list[dict], values: dict[str, int], tag_to_string,
+ regular_cases: dict[str, str],
+ accepted: list[str]) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
+ """Build the (go_name, go_value) lists for the generated constants.
+
+ Returns the tags Kiwi can emit and, separately, the input-only aliases that
+ upstream's toPOSTag accepts but never produces.
+ """
+ emitted: list[tuple[str, str]] = []
+ seen_names: set[str] = set()
+ seen_values: set[str] = set()
+
+ def add(go_name: str, go_value: str, target: list[tuple[str, str]]) -> None:
+ if go_name in seen_names:
+ return
+ seen_names.add(go_name)
+ seen_values.add(go_value)
+ target.append((go_name, go_value))
+
+ for tag in tags:
+ cpp_name = tag["name"]
+ if cpp_name in SKIPPED_ENUMERATORS:
+ continue
+
+ go_value = tag_to_string(values[cpp_name])
+ if go_value is None:
+ continue
+
+ add(go_constant_name(cpp_name), go_value, emitted)
+ # `-R` variants are only reachable through tagRToString, so they are not
+ # derivable from the enum alone.
+ for cpp_name, text in regular_cases.items():
+ add(f"POS_{cpp_name.upper()}_R", text, emitted)
+
+ # Strings toPOSTag accepts as input but tagToString never returns, such as
+ # "V" and "A" for POSTag::p or "NF"/"NV"/"NA"/"UNK" for POSTag::unknown.
+ aliases: list[tuple[str, str]] = []
+ for text in accepted:
+ if text in seen_values:
+ continue
+ go_name = PUNCTUATION_TAG_NAMES.get(text, f"POS_{text.upper()}")
+ add(go_name, text, aliases)
+
+ return emitted, aliases
+
+
+def generate_go_file(emitted: list[tuple[str, str]], aliases: list[tuple[str, str]],
+ version: str) -> str:
+ """Generate Go source file with POS type definitions."""
+ entries = emitted + aliases
lines = []
lines.append("// Code generated by scripts/extract_postags.py; DO NOT EDIT.")
lines.append(f"// Source: Kiwi {version}")
@@ -136,50 +277,47 @@ def generate_go_file(tags: list[dict], tag_strings: list[str], version: str) ->
lines.append("")
lines.append("const (")
- # Track which tags we've added and collect for alignment
- added_tags = set()
- tag_entries = []
-
- for i, tag in enumerate(tags):
- cpp_name = tag["name"]
- tag_string = string_lookup.get(i, cpp_name.upper())
+ # gofmt aligns each run of declarations separated by a blank line on its
+ # own, so each group is padded to its own longest name.
+ def declare_group(group: list[tuple[str, str]]) -> list[str]:
+ width = max(len(name) for name, _ in group)
+ return [
+ f'\t{go_name}{" " * (width - len(go_name) + 1)}POSType = "{go_value}"'
+ for go_name, go_value in group
+ ]
- result = map_cpp_to_go_tag(cpp_name, tag_string)
- if result is None:
- continue
-
- go_name, go_value = result
- if go_name in added_tags:
- continue
-
- added_tags.add(go_name)
- tag_entries.append((go_name, go_value))
-
- # Calculate max name length for alignment
- max_name_len = max(len(name) for name, _ in tag_entries)
+ lines.extend(declare_group(emitted))
+ lines.append("")
+ lines.append("\t// Accepted by toPOSTag but never produced by Kiwi itself.")
+ lines.extend(declare_group(aliases))
- # Generate aligned output
- for go_name, go_value in tag_entries:
- padding = " " * (max_name_len - len(go_name) + 1)
- lines.append(f'\t{go_name}{padding}POSType = "{go_value}"')
+ lines.append(")")
+ lines.append("")
+ # Backwards-compatible aliases.
+ lines.append("// Deprecated: these aliases are kept for backwards compatibility.")
+ lines.append("const (")
+ alias_len = max(len(old) for old, _ in COMPAT_ALIASES)
+ for old_name, new_name in COMPAT_ALIASES:
+ padding = " " * (alias_len - len(old_name) + 1)
+ lines.append(f"\t{old_name}{padding}= {new_name}")
lines.append(")")
lines.append("")
- # Generate isValid function
+ # Generate isValid function. This mirrors which strings upstream's toPOSTag
+ # accepts, plus the strings tagToString/tagRToString can return. A Go switch
+ # requires distinct case values, and several constants are aliases sharing
+ # one string, so deduplicate by value.
lines.append("func (p POSType) isValid() bool {")
lines.append("\tswitch p {")
lines.append("\tcase")
valid_tags = []
- for tag in tags:
- cpp_name = tag["name"]
- result = map_cpp_to_go_tag(cpp_name, "")
- if result is None:
- continue
- go_name = result[0]
- if go_name not in added_tags:
+ seen_values: set[str] = set()
+ for go_name, go_value in entries:
+ if go_value in seen_values:
continue
+ seen_values.add(go_value)
valid_tags.append(go_name)
lines.append(",\n".join(f"\t\t{tag}" for tag in valid_tags) + ":")
@@ -200,7 +338,7 @@ def generate_go_file(tags: list[dict], tag_strings: list[str], version: str) ->
lines.append("\treturn pos, nil")
lines.append("}")
- return "\n".join(lines)
+ return "\n".join(lines) + "\n"
def main():
@@ -218,18 +356,48 @@ def main():
print(f"Fetching {utils_url}")
utils_source = fetch_file(utils_url)
+ strutils_url = KIWI_STRUTILS_H_URL.format(version=version)
+ print(f"Fetching {strutils_url}")
+ strutils_source = fetch_file(strutils_url)
+
# Extract enum values
print("Parsing POSTag enum...")
tags = extract_enum_values(types_source, "POSTag")
print(f"Found {len(tags)} enum values")
- # Extract tag strings
+ values = resolve_enum_values(tags)
+ if "irregular" not in values:
+ raise SystemExit("POSTag::irregular not found; cannot decode irregular tags")
+
+ # Extract the conversion tables.
print("Parsing tag strings...")
- tag_strings = extract_tag_strings(utils_source)
+ tag_to_string_body = extract_function_body(utils_source, r"const\s+char\s*\*\s*tagToString\s*\(")
+ tag_strings = extract_tag_strings(tag_to_string_body)
+ if not tag_strings:
+ raise SystemExit("tagToString tag table not found")
print(f"Found {len(tag_strings)} tag strings")
+ irregular_cases = extract_case_returns(tag_to_string_body)
+
+ tag_r_to_string_body = extract_function_body(utils_source, r"const\s+char\s*\*\s*tagRToString\s*\(")
+ regular_cases = extract_case_returns(tag_r_to_string_body)
+ print(f"Found {len(regular_cases)} regular-conjugation tags")
+
+ # toPOSTag defines which strings upstream accepts as input, which is the
+ # role ParsePOSType plays on the Go side.
+ to_pos_tag_body = extract_function_body(strutils_source, r"inline\s+POSTag\s+toPOSTag\s*\(\s*std::u16string_view")
+ accepted = extract_accepted_tags(to_pos_tag_body)
+ if not accepted:
+ raise SystemExit("toPOSTag not found; cannot determine accepted tag strings")
+ print(f"Found {len(accepted)} accepted tag strings")
+
+ tag_to_string = make_tag_to_string(
+ tag_strings, irregular_cases, values, values["irregular"], values["max"]
+ )
+ emitted, aliases = build_tag_entries(tags, values, tag_to_string, regular_cases, accepted)
+
# Generate Go file
- go_content = generate_go_file(tags, tag_strings, version)
+ go_content = generate_go_file(emitted, aliases, version)
# Write output
output_path = Path("postype_generated.go")
@@ -238,13 +406,17 @@ def main():
# Print summary
print("\nExtracted tags:")
- for i, tag in enumerate(tags):
- tag_str = tag_strings[i] if i < len(tag_strings) else "?"
- result = map_cpp_to_go_tag(tag["name"], tag_str)
- if result:
- print(f" {tag['name']:20s} -> {result[0]:20s} = \"{result[1]}\"")
- else:
- print(f" {tag['name']:20s} -> (skipped)")
+ for tag in tags:
+ cpp_name = tag["name"]
+ text = None if cpp_name in SKIPPED_ENUMERATORS else tag_to_string(values[cpp_name])
+ if text is None:
+ print(f" {cpp_name:20s} -> (no tag string; skipped)")
+ continue
+ print(f' {cpp_name:20s} -> {go_constant_name(cpp_name):20s} = "{text}"')
+ for cpp_name, text in regular_cases.items():
+ print(f' {cpp_name + " (R)":20s} -> {"POS_" + cpp_name.upper() + "_R":20s} = "{text}"')
+ for go_name, text in aliases:
+ print(f' {"(toPOSTag only)":20s} -> {go_name:20s} = "{text}"')
if __name__ == "__main__":