feat: add POS tag sync workflow with C++ upstream (closes #40)

.github/workflows/sync-postypes.yaml

@@ -41,13 +41,13 @@ jobs:
         id: get_version
         run: |
           if [ -n "${{ github.event.inputs.kiwi_version }}" ]; then
-            echo "version=${{ github.event.inputs.kiwi_version }}" >> $GITHUB_OUTPUT
+            VERSION="${{ github.event.inputs.kiwi_version }}"
           else
             # Get latest release from GitHub API
             VERSION=$(curl -s https://api.github.com/repos/bab2min/Kiwi/releases/latest | jq -r '.tag_name')
-            echo "version=$VERSION" >> $GITHUB_OUTPUT
           fi
-          echo "Kiwi version: $(cat $GITHUB_OUTPUT | grep version | cut -d= -f2)"
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "Kiwi version: $VERSION"
 
       - name: Extract POS tags
         id: extract
@@ -89,6 +89,15 @@ jobs:
           # Update Makefile version
           sed -i "s/KIWI_VERSION := .*/KIWI_VERSION := ${{ steps.get_version.outputs.version }}/" Makefile
 
+      # PRs opened with GITHUB_TOKEN do not trigger CI, so the generated code is
+      # verified here instead. Without this a non-compiling postype.go can be
+      # proposed and merged unnoticed.
+      - name: Verify generated code
+        if: steps.extract.outputs.changed == 'true'
+        run: |
+          make install-kiwi
+          make test
+
       - name: Create Pull Request
         if: steps.extract.outputs.changed == 'true'
         uses: peter-evans/create-pull-request@v7

.gitignore

@@ -26,3 +26,6 @@ ModelGenerator/
 
 # Python virtual environment
 .venv/
+
+# Generated by scripts/extract_postags.py before it replaces postype.go
+postype_generated.go

Makefile

@@ -25,18 +25,27 @@ format:
 	# go install mvdan.cc/gofumpt@latest
 	gofumpt -l -w .
 
+VENV := .venv
+VENV_PYTHON := $(VENV)/bin/python
+
+# `source` is not available under /bin/sh on Debian-family systems, so the
+# venv interpreter is invoked directly instead of activating the venv.
+$(VENV_PYTHON):
+	python3 -m venv $(VENV)
+	$(VENV)/bin/pip install --quiet tree-sitter tree-sitter-cpp
+
 .PHONY: sync-postypes
-sync-postypes:
+sync-postypes: $(VENV_PYTHON)
 	@echo "Extracting POS tags from Kiwi $(KIWI_VERSION)..."
-	source .venv/bin/activate && python scripts/extract_postags.py $(KIWI_VERSION)
+	$(VENV_PYTHON) scripts/extract_postags.py $(KIWI_VERSION)
 	@echo "Generated postype_generated.go"
 	@echo "Comparing with current postype.go..."
 	@diff -u postype.go postype_generated.go || true
 	@echo "To apply changes, run: mv postype_generated.go postype.go"
 
 .PHONY: check-postypes
-check-postypes:
-	@source .venv/bin/activate && python scripts/extract_postags.py $(KIWI_VERSION)
+check-postypes: $(VENV_PYTHON)
+	@$(VENV_PYTHON) scripts/extract_postags.py $(KIWI_VERSION)
 	@if diff -q postype.go postype_generated.go > /dev/null 2>&1; then \
 		echo "POS types are in sync with Kiwi $(KIWI_VERSION)"; \
 	else \

postype_test.go

@@ -25,6 +25,47 @@ func TestParsePOSType(t *testing.T) {
 			want:    POS_UNKNOWN,
 			wantErr: true,
 		},
+		// kiwi_res_tag is implemented with tagRToString, so the -R variants
+		// reach Go for regular conjugations. See #39.
+		{
+			name:    "VV-R is a POSType",
+			arg:     "VV-R",
+			want:    POS_VV_R,
+			wantErr: false,
+		},
+		{
+			name:    "XSA-R is a POSType",
+			arg:     "XSA-R",
+			want:    POS_XSA_R,
+			wantErr: false,
+		},
+		{
+			name:    "VV-I is a POSType",
+			arg:     "VV-I",
+			want:    POS_VV_I,
+			wantErr: false,
+		},
+		// toPOSTag accepts these as input even though Kiwi never emits them.
+		{
+			name:    "V is a POSType",
+			arg:     "V",
+			want:    POS_V,
+			wantErr: false,
+		},
+		{
+			name:    "UNK is a POSType",
+			arg:     "UNK",
+			want:    POS_UNK,
+			wantErr: false,
+		},
+		// toPOSTag has no entry for "@"; it is the sentinel tagToString falls
+		// back to for values it is never meant to be called with.
+		{
+			name:    "@ is not a valid POSType",
+			arg:     "@",
+			want:    POS_UNKNOWN,
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -39,3 +80,26 @@ func TestParsePOSType(t *testing.T) {
 		})
 	}
 }
+
+// TestAnalyzeRegularConjugation guards against #39. Kiwi tags a regular verb
+// whose stem ends in ㄷ/ㅂ/ㅅ as VV-R rather than VV, so dropping the -R
+// constants makes Analyze fail on ordinary sentences.
+func TestAnalyzeRegularConjugation(t *testing.T) {
+	kiwi, err := New("./base", WithNumThread(1))
+	assert.NoError(t, err)
+
+	for _, sentence := range []string{"편지를 받았다", "나는 공을 잡았다", "그는 크게 웃었다"} {
+		t.Run(sentence, func(t *testing.T) {
+			res, err := kiwi.Analyze(sentence)
+			assert.NoError(t, err)
+
+			var tags []POSType
+			for _, result := range res {
+				for _, token := range result.Tokens {
+					tags = append(tags, token.Tag)
+				}
+			}
+			assert.Contains(t, tags, POS_VV_R)
+		})
+	}
+}