diff --git a/.dockerignore b/.dockerignore index 1fe3b45..46c8380 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,8 +19,9 @@ requirements-dev.txt secrets data_science/** !data_science/SMSModel/ -!data_science/SMSModel/phishing_model_artifact.pkl -!data_science/SMSModel/phishing_vectorizer.pkl +!data_science/SMSModel/artifacts/ +!data_science/SMSModel/artifacts/phishing_model_artifact.pkl +!data_science/SMSModel/artifacts/phishing_vectorizer.pkl *.ipynb *.csv *.png diff --git a/.env.example b/.env.example index 8050c4b..f9eeb8a 100644 --- a/.env.example +++ b/.env.example @@ -9,5 +9,5 @@ MOCK_SECURITY_API=false RABBITMQ_URL=amqp://safefam:safefam-local@localhost:5672/ RABBITMQ_CONSUMER_ENABLED=false -NAIVE_BAYES_MODEL_PATH=data_science/SMSModel/phishing_model_artifact.pkl -NAIVE_BAYES_VECTORIZER_PATH=data_science/SMSModel/phishing_vectorizer.pkl +NAIVE_BAYES_MODEL_PATH=data_science/SMSModel/artifacts/phishing_model_artifact.pkl +NAIVE_BAYES_VECTORIZER_PATH=data_science/SMSModel/artifacts/phishing_vectorizer.pkl diff --git a/Dockerfile b/Dockerfile index 6845684..370d7e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,9 +10,13 @@ ENV NAIVE_BAYES_VECTORIZER_PATH=/app/models/phishing_vectorizer.pkl COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt +# Fail the image build early when the Kiwi wheel cannot be imported or its +# morphological analyzer cannot be initialized in the target architecture. +RUN python -c "from kiwipiepy import Kiwi; assert Kiwi().tokenize('installation check')" + COPY app app -COPY data_science/SMSModel/phishing_model_artifact.pkl models/phishing_model_artifact.pkl -COPY data_science/SMSModel/phishing_vectorizer.pkl models/phishing_vectorizer.pkl +COPY data_science/SMSModel/artifacts/phishing_model_artifact.pkl models/phishing_model_artifact.pkl +COPY data_science/SMSModel/artifacts/phishing_vectorizer.pkl models/phishing_vectorizer.pkl RUN useradd --create-home --shell /usr/sbin/nologin safefam \ && chown -R safefam:safefam /app diff --git a/SCORING_PIPELINE_CHANGES.md b/SCORING_PIPELINE_CHANGES.md index b41a3be..3659d8c 100644 --- a/SCORING_PIPELINE_CHANGES.md +++ b/SCORING_PIPELINE_CHANGES.md @@ -23,7 +23,7 @@ **`app/service/security/naive_bayes_text_analyzer.py`** (신규) -- 기존에 학습돼 저장소에 커밋되어 있던 `data_science/SMSModel/phishing_model_artifact.pkl`(CalibratedClassifierCV + ComplementNB), `phishing_vectorizer.pkl`을 로드 +- 기존에 학습돼 저장소에 커밋되어 있던 `data_science/SMSModel/artifacts/phishing_model_artifact.pkl`(CalibratedClassifierCV + ComplementNB), `phishing_vectorizer.pkl`을 로드 - 전처리(URL/전화번호/금액 마스킹, 6개 구조적 피처)는 학습 스크립트(`train_sms.py`)와 동일하게 재구현 — 학습/서빙 피처 불일치 방지 - 모델 로드 실패 시 `UNKNOWN` 등급 + 에러 메시지로 fail-safe 처리 (SAFE로 오판하지 않음) - `requirements.txt`에 `scikit-learn`, `scipy`, `numpy`, `joblib` 추가 diff --git a/app/analysis/text/naive_bayes_analyzer.py b/app/analysis/text/naive_bayes_analyzer.py index 93305f2..0da116b 100644 --- a/app/analysis/text/naive_bayes_analyzer.py +++ b/app/analysis/text/naive_bayes_analyzer.py @@ -1,94 +1,85 @@ -import re +# 사전 학습된 Naive Bayes 모델을 이용한 SMS 피싱 분석기 +from __future__ import annotations + import logging +import numpy as np +from scipy.sparse import csr_matrix, hstack + from app.analysis.risk_policy import determine_text_risk_grade +from app.analysis.text.preprocessing import ( + extract_struct_features, + normalize_text, +) from app.core.config import settings + logger = logging.getLogger(__name__) + MODEL_PATH = settings.NAIVE_BAYES_MODEL_PATH VECTORIZER_PATH = settings.NAIVE_BAYES_VECTORIZER_PATH -# --- 전처리 정규식 : data_science/SMSModel/train_sms.py의 정규화/피처 추출 로직과 반드시 동일하게 유지 --- -# (학습 시 벡터라이저가 본 입력 분포와 서빙 시 입력 분포가 어긋나면 모델이 무의미해짐) -_RE_URL = re.compile(r"(?i)(? str: - parts = [] - last_end = 0 - for m in _RE_URL.finditer(text): - parts.append(_mask_pii(text[last_end:m.start()])) - parts.append("[URL]") - last_end = m.end() - parts.append(_mask_pii(text[last_end:])) - text = "".join(parts) - text = _RE_AMOUNT.sub("[AMOUNT]", text) - text = _RE_FORMAT_ARTIFACT.sub(" ", text) - return re.sub(r"\s+", " ", text).strip() - -def _mask_pii(text: str) -> str: - text = _RE_RRN.sub("[RRN]", text) - text = _RE_CARD.sub("[CARD]", text) - text = _RE_PHONE.sub("[PHONE]", text) - text = _RE_ACCOUNT.sub("[ACCOUNT]", text) - text = _RE_EMAIL.sub("[EMAIL]", text) - return text - - -def _extract_struct_features(text: str) -> list: - return [ - int(bool(_RE_URL.search(text))), - int(bool(_RE_SHORT_URL.search(text))), - int(bool(_RE_PHONE.search(text) or "[PHONE]" in text)), - int(bool(_RE_AMOUNT.search(text))), - int(bool(_RE_WEB_TAG.search(text))), - int(len(text) > 100), - ] +_load_attempted = False def _load_artifacts() -> None: - """FastAPI 프로세스 당 1회만 시도. 실패 시 재시도하지 않고 fail-safe 응답으로 대체.""" - global _model, _vectorizer, _threshold, _classes, _load_error, _load_attempted + """ + 모델과 벡터라이저를 프로세스당 한 번만 로드 + + 로드에 실패해도 API 서버 전체를 중단시키지 않고, 이후 분석 요청에서 + UNKNOWN 결과를 반환할 수 있도록 오류 상태만 저장 + """ + global _model + global _vectorizer + global _threshold + global _classes + global _load_error + global _load_attempted if _load_attempted: return + _load_attempted = True try: import joblib artifact = joblib.load(MODEL_PATH) + vectorizer = joblib.load(VECTORIZER_PATH) + + # 모든 값이 정상적으로 읽힌 이후 전역 상태를 갱신 _model = artifact["model"] _threshold = artifact["threshold"] _classes = artifact["classes"] - _vectorizer = joblib.load(VECTORIZER_PATH) - logger.info(f"[NaiveBayes] 모델 로드 완료 (threshold={_threshold})") + _vectorizer = vectorizer + _load_error = None + + logger.info( + "[NaiveBayes] 모델 로드 완료 (threshold=%s)", + _threshold, + ) except Exception as exception: _load_error = type(exception).__name__ + logger.error( "[NaiveBayes] 모델 로드 실패. error_type=%s", _load_error, @@ -96,36 +87,57 @@ def _load_artifacts() -> None: def is_model_loaded() -> bool: + """Naive Bayes 모델을 사용할 수 있는지 반환""" _load_artifacts() return _model is not None -# 사전 학습된 나이브 베이즈(ComplementNB + isotonic 보정) 모델로 문자 메시지의 1차 위험도를 산출 async def analyze_text_with_naive_bayes(text: str) -> dict: + """사전 학습된 Naive Bayes 모델로 SMS의 피싱 위험도를 분석""" _load_artifacts() if _model is None: return { "engine": "naive_bayes", "is_available": False, - "result": dict(DEFAULT_ANALYSIS_RESULT, error_message=_load_error or DEFAULT_ANALYSIS_RESULT["error_message"]) + "result": dict( + DEFAULT_ANALYSIS_RESULT, + error_message=( + _load_error + or DEFAULT_ANALYSIS_RESULT["error_message"] + ), + ), } try: - import numpy as np - from scipy.sparse import csr_matrix, hstack + # 학습과 동일한 공통 전처리를 적용 + normalized_text = normalize_text(text) - text_norm = _normalize_text(text) - struct = np.array([_extract_struct_features(text)]) + struct_features = np.asarray( + [extract_struct_features(text)], + dtype=np.int8, + ) - X_text = _vectorizer.transform([text_norm]) - X = hstack([X_text, csr_matrix(struct)]) + # 기존 artifact가 기대하는 입력 구조를 유지 + text_features = _vectorizer.transform([normalized_text]) + feature_matrix = hstack( + [ + text_features, + csr_matrix(struct_features), + ] + ) - phishing_idx = _classes.index("phishing") - prob_phishing = _model.predict_proba(X)[0][phishing_idx] - risk_score = int(prob_phishing * 100) + phishing_index = _classes.index("phishing") + phishing_probability = _model.predict_proba( + feature_matrix + )[0][phishing_index] - logger.info(f"[NaiveBayes] 문자 분석 완료 - 위험도 점수: {risk_score}") + risk_score = int(phishing_probability * 100) + + logger.info( + "[NaiveBayes] 문자 분석 완료 - 위험도 점수: %s", + risk_score, + ) return { "engine": "naive_bayes", @@ -133,17 +145,23 @@ async def analyze_text_with_naive_bayes(text: str) -> dict: "result": { "grade": determine_text_risk_grade(risk_score), "risk_score": risk_score, - "is_suspected_phishing": bool(prob_phishing >= _threshold), - "error_message": None - } + "is_suspected_phishing": bool( + phishing_probability >= _threshold + ), + "error_message": None, + }, } except Exception as exception: logger.error( "[NaiveBayes] 추론 중 비정상 오류 발생. error_type=%s", type(exception).__name__, ) + return { "engine": "naive_bayes", "is_available": False, - "result": dict(DEFAULT_ANALYSIS_RESULT, error_message="Inference Error") - } + "result": dict( + DEFAULT_ANALYSIS_RESULT, + error_message="Inference Error", + ), + } \ No newline at end of file diff --git a/app/analysis/text/preprocessing.py b/app/analysis/text/preprocessing.py new file mode 100644 index 0000000..1141d8f --- /dev/null +++ b/app/analysis/text/preprocessing.py @@ -0,0 +1,126 @@ +"""SMS 모델의 학습과 추론에서 공통으로 사용하는 전처리 로직.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from typing import TYPE_CHECKING + +import numpy as np + +if TYPE_CHECKING: + import pandas as pd + + +# 학습된 모델은 아래 피처 순서에 의존하므로 순서를 변경하면 안 됩니다. +STRUCT_FEATURE_NAMES: tuple[str, ...] = ( + "has_url", + "has_short_url", + "has_phone", + "has_amount", + "has_web_tag", + "is_long_text", +) + +URL_PATTERN = re.compile( + r"(?i)(? str: + """개인정보를 Spring PiiMaskingService와 같은 순서로 마스킹합니다.""" + if not isinstance(text, str): + raise TypeError("text must be a string") + + masked = RRN_PATTERN.sub("[RRN]", text) + masked = CARD_PATTERN.sub("[CARD]", masked) + masked = PHONE_PATTERN.sub("[PHONE]", masked) + masked = ACCOUNT_PATTERN.sub("[ACCOUNT]", masked) + masked = EMAIL_PATTERN.sub("[EMAIL]", masked) + return masked + + +def normalize_text(text: str) -> str: + """URL·개인정보·금액을 치환하고 공백을 정리한 모델 입력을 만듭니다.""" + if not isinstance(text, str): + raise TypeError("text must be a string") + + parts: list[str] = [] + last_end = 0 + + # URL 내부 숫자가 전화번호나 계좌번호로 오인되지 않도록 URL부터 분리합니다. + for match in URL_PATTERN.finditer(text): + parts.append(mask_pii(text[last_end:match.start()])) + parts.append("[URL]") + last_end = match.end() + + parts.append(mask_pii(text[last_end:])) + normalized = "".join(parts) + normalized = AMOUNT_PATTERN.sub("[AMOUNT]", normalized) + normalized = FORMAT_ARTIFACT_PATTERN.sub(" ", normalized) + return WHITESPACE_PATTERN.sub(" ", normalized).strip() + + +def extract_struct_features(text: str, *, has_url: bool | None = None) -> list[int]: + """단일 원문 SMS에서 고정 순서의 구조 피처 6개를 추출합니다.""" + if not isinstance(text, str): + raise TypeError("text must be a string") + + detected_url = bool(URL_PATTERN.search(text)) + url_feature = detected_url if has_url is None else bool(has_url) + + return [ + int(url_feature), + int(bool(SHORT_URL_PATTERN.search(text))), + int(bool(PHONE_PATTERN.search(text) or "[PHONE]" in text)), + int(bool(AMOUNT_PATTERN.search(text) or "[AMOUNT]" in text)), + int(bool(WEB_TAG_PATTERN.search(text))), + int(len(text) > 100), + ] + + +def extract_struct_feature_matrix( + texts: Iterable[str] | pd.Series, + has_urls: Iterable[bool] | pd.Series | None = None, +) -> np.ndarray: + """여러 원문 SMS의 구조 피처를 ``(n_samples, 6)`` 배열로 반환합니다.""" + text_list = list(texts) + + if has_urls is None: + url_list: list[bool | None] = [None] * len(text_list) + else: + url_list = list(has_urls) + if len(text_list) != len(url_list): + raise ValueError( + "texts and has_urls must contain the same number of items" + ) + + rows = [ + extract_struct_features(text, has_url=has_url) + for text, has_url in zip(text_list, url_list, strict=True) + ] + + if not rows: + return np.empty((0, len(STRUCT_FEATURE_NAMES)), dtype=np.int8) + return np.asarray(rows, dtype=np.int8) diff --git a/app/core/config.py b/app/core/config.py index 5c55ea5..5bf70a3 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -18,10 +18,10 @@ class Settings(BaseSettings): MOCK_SECURITY_API: bool = False NAIVE_BAYES_MODEL_PATH: Path = Path( - "data_science/SMSModel/phishing_model_artifact.pkl" + "data_science/SMSModel/artifacts/phishing_model_artifact.pkl" ) NAIVE_BAYES_VECTORIZER_PATH: Path = Path( - "data_science/SMSModel/phishing_vectorizer.pkl" + "data_science/SMSModel/artifacts/phishing_vectorizer.pkl" ) GEMINI_TIMEOUT_SECONDS: float = Field( diff --git a/data_science/SMSModel/README.md b/data_science/SMSModel/README.md new file mode 100644 index 0000000..5301d21 --- /dev/null +++ b/data_science/SMSModel/README.md @@ -0,0 +1,23 @@ +# SMS Model Workspace + +| Path | Purpose | +|---|---| +| `train_sms.py` | Naive Bayes training entry point | +| `artifacts/` | Versioned model and vectorizer files used by the API | +| `dataset_splitting/` | Leakage-safe train/validation/test splitting | +| `template_grouping/` | Duplicate and similar-message grouping | +| `splits/` | Reproducible split manifests | +| `reporting/` | Dataset report generation code | +| `reports/` | Generated summaries, metrics, and feature analysis data | +| `reports/figures/` | Generated plots and figures | +| `SMSDataModel.ipynb` | Exploratory analysis notebook | + +Run training from the repository root: + +```bash +python -m data_science.SMSModel.train_sms +``` + +The committed split manifest fixes the final test set. Do not overwrite it during +routine training. Create a new manifest version when the dataset, preprocessing, +template grouping, or split policy intentionally changes. diff --git a/data_science/SMSModel/SMSDataModel.ipynb b/data_science/SMSModel/SMSDataModel.ipynb index 0d62813..8217d30 100644 --- a/data_science/SMSModel/SMSDataModel.ipynb +++ b/data_science/SMSModel/SMSDataModel.ipynb @@ -21,8 +21,8 @@ "SMSModel/\n", "├── train_sms.py # 학습 스크립트\n", "├── SMSData.ipynb # 시각화 전용 (이 파일)\n", - "├── phishing_model_artifact.pkl # 학습 완료 후 생성\n", - "└── phishing_vectorizer.pkl # 학습 완료 후 생성\n", + "├── artifacts/phishing_model_artifact.pkl # 학습 완료 후 생성\n", + "└── artifacts/phishing_vectorizer.pkl # 학습 완료 후 생성\n", "```\n" ] }, @@ -228,9 +228,9 @@ "ax2.xaxis.set_major_formatter(mticker.PercentFormatter(xmax=1))\n", "\n", "plt.tight_layout()\n", - "plt.savefig(\"risk_distribution_fig1.png\", dpi=150, bbox_inches=\"tight\")\n", + "plt.savefig(\"reports/figures/risk_distribution_fig1.png\", dpi=150, bbox_inches=\"tight\")\n", "plt.show()\n", - "print(\"[Save] risk_distribution_fig1.png\")\n" + "print(\"[Save] reports/figures/risk_distribution_fig1.png\")\n" ] }, { @@ -312,9 +312,9 @@ "ax2.yaxis.set_major_formatter(mticker.PercentFormatter())\n", "\n", "plt.tight_layout()\n", - "plt.savefig(\"risk_distribution_fig2.png\", dpi=150, bbox_inches=\"tight\")\n", + "plt.savefig(\"reports/figures/risk_distribution_fig2.png\", dpi=150, bbox_inches=\"tight\")\n", "plt.show()\n", - "print(\"[Save] risk_distribution_fig2.png\")\n" + "print(\"[Save] reports/figures/risk_distribution_fig2.png\")\n" ] }, { @@ -511,9 +511,9 @@ "axes[1].axvline(0, color=\"black\", linewidth=0.8)\n", "\n", "plt.tight_layout()\n", - "plt.savefig(\"feature_importance.png\", dpi=150, bbox_inches=\"tight\")\n", + "plt.savefig(\"reports/figures/feature_importance.png\", dpi=150, bbox_inches=\"tight\")\n", "plt.show()\n", - "print(\"[Save] feature_importance.png\")\n" + "print(\"[Save] reports/figures/feature_importance.png\")\n" ] } ], diff --git a/data_science/SMSModel/phishing_model_artifact.pkl b/data_science/SMSModel/artifacts/phishing_model_artifact.pkl similarity index 100% rename from data_science/SMSModel/phishing_model_artifact.pkl rename to data_science/SMSModel/artifacts/phishing_model_artifact.pkl diff --git a/data_science/SMSModel/phishing_vectorizer.pkl b/data_science/SMSModel/artifacts/phishing_vectorizer.pkl similarity index 100% rename from data_science/SMSModel/phishing_vectorizer.pkl rename to data_science/SMSModel/artifacts/phishing_vectorizer.pkl diff --git a/data_science/SMSModel/dataset_splitting/__init__.py b/data_science/SMSModel/dataset_splitting/__init__.py new file mode 100644 index 0000000..0c5374d --- /dev/null +++ b/data_science/SMSModel/dataset_splitting/__init__.py @@ -0,0 +1,19 @@ +"""SMS 데이터 분할 공개 API""" +from .config import DatasetSplitConfig +from .manifest import ( + build_split_manifest, + load_split_manifest, + save_split_manifest, +) +from .splitter import DatasetSplits, split_grouped_dataset +from .validation import validate_dataset_splits + +__all__ = [ + "DatasetSplitConfig", + "DatasetSplits", + "build_split_manifest", + "load_split_manifest", + "save_split_manifest", + "split_grouped_dataset", + "validate_dataset_splits", +] diff --git a/data_science/SMSModel/dataset_splitting/config.py b/data_science/SMSModel/dataset_splitting/config.py new file mode 100644 index 0000000..8ebe6b4 --- /dev/null +++ b/data_science/SMSModel/dataset_splitting/config.py @@ -0,0 +1,33 @@ +"""SMS train/validation/test 분할 설정.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DatasetSplitConfig: + """그룹 보존 분할을 동일하게 재현하기 위한 불변 설정.""" + + train_size: float = 0.70 + val_size: float = 0.15 + test_size: float = 0.15 + random_state: int = 42 + candidate_count: int = 500 + group_column: str = "template_group_id" + label_column: str = "label" + fingerprint_column: str = "text_fingerprint" + + def __post_init__(self) -> None: + total_size = self.train_size + self.val_size + self.test_size + if abs(total_size - 1.0) > 1e-9: + raise ValueError("train_size, val_size and test_size must sum to 1.0") + for name, value in ( + ("train_size", self.train_size), + ("val_size", self.val_size), + ("test_size", self.test_size), + ): + if not 0.0 < value < 1.0: + raise ValueError(f"{name} must be between 0 and 1") + if self.candidate_count <= 0: + raise ValueError("candidate_count must be greater than 0") diff --git a/data_science/SMSModel/dataset_splitting/manifest.py b/data_science/SMSModel/dataset_splitting/manifest.py new file mode 100644 index 0000000..c71e51d --- /dev/null +++ b/data_science/SMSModel/dataset_splitting/manifest.py @@ -0,0 +1,134 @@ +"""재현 가능한 SMS split manifest 저장 및 적용.""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + +from .config import DatasetSplitConfig +from .splitter import DatasetSplits +from .validation import validate_dataset_splits + + +MANIFEST_COLUMNS = [ + "text_fingerprint", + "template_group_id", + "split", + "label", + "type", +] +ALLOWED_SPLITS = {"train", "validation", "test"} + + +def build_split_manifest(splits: DatasetSplits) -> pd.DataFrame: + """원문 없이 재현에 필요한 배정 정보만 정렬해 반환합니다.""" + combined = pd.concat( + [splits.train, splits.validation, splits.test], + ignore_index=True, + ) + missing = set(MANIFEST_COLUMNS) - set(combined.columns) + if missing: + raise ValueError(f"cannot build manifest; missing columns: {missing}") + return ( + combined[MANIFEST_COLUMNS] + .sort_values(["split", "template_group_id", "text_fingerprint"]) + .reset_index(drop=True) + ) + + +def save_split_manifest( + splits: DatasetSplits, + path: Path, + *, + overwrite: bool = False, +) -> pd.DataFrame: + """기존 파일은 명시적 요청 없이는 덮어쓰지 않고 CSV를 저장합니다.""" + path = Path(path) + if path.exists() and not overwrite: + raise FileExistsError( + f"split manifest already exists: {path}. " + "Use a new version or pass overwrite=True intentionally." + ) + manifest = build_split_manifest(splits) + path.parent.mkdir(parents=True, exist_ok=True) + manifest.to_csv(path, index=False, encoding="utf-8", lineterminator="\n") + return manifest + + +def apply_split_manifest( + df: pd.DataFrame, + manifest: pd.DataFrame, + *, + config: DatasetSplitConfig | None = None, +) -> DatasetSplits: + """fingerprint를 키로 저장된 배정을 현재 데이터에 안전하게 적용합니다.""" + config = config or DatasetSplitConfig() + required = { + config.fingerprint_column, + config.group_column, + config.label_column, + "split", + } + missing = required - set(manifest.columns) + if missing: + raise ValueError(f"missing manifest columns: {missing}") + if manifest[config.fingerprint_column].duplicated().any(): + raise ValueError("manifest contains duplicate text fingerprints") + if set(manifest["split"].astype(str)) != ALLOWED_SPLITS: + raise ValueError("manifest must contain only train, validation and test") + if manifest.groupby(config.group_column)["split"].nunique().max() > 1: + raise ValueError("manifest assigns one template group to multiple splits") + + current_keys = set(df[config.fingerprint_column].astype(str)) + manifest_keys = set(manifest[config.fingerprint_column].astype(str)) + if current_keys != manifest_keys: + raise ValueError("dataset does not match split manifest fingerprints") + + # 현재 그룹 및 label이 manifest 생성 당시와 같은지도 확인합니다. + verification = df[ + [config.fingerprint_column, config.group_column, config.label_column] + ].merge( + manifest[ + [config.fingerprint_column, config.group_column, config.label_column] + ], + on=config.fingerprint_column, + suffixes=("_current", "_manifest"), + validate="one_to_one", + ) + for column in (config.group_column, config.label_column): + if not ( + verification[f"{column}_current"].astype(str) + == verification[f"{column}_manifest"].astype(str) + ).all(): + raise ValueError(f"dataset {column} does not match split manifest") + + merged = df.merge( + manifest[[config.fingerprint_column, "split"]], + on=config.fingerprint_column, + how="left", + validate="one_to_one", + ) + splits = DatasetSplits( + train=merged[merged["split"] == "train"].copy().reset_index(drop=True), + validation=merged[merged["split"] == "validation"].copy().reset_index( + drop=True + ), + test=merged[merged["split"] == "test"].copy().reset_index(drop=True), + ) + validate_dataset_splits(df, splits, config=config) + return splits + + +def load_split_manifest( + df: pd.DataFrame, + path: Path, + *, + config: DatasetSplitConfig | None = None, +) -> DatasetSplits: + """CSV manifest를 읽어 현재 데이터에 적용합니다.""" + path = Path(path) + if not path.exists(): + raise FileNotFoundError(f"split manifest does not exist: {path}") + manifest = pd.read_csv(path, dtype="string") + return apply_split_manifest(df, manifest, config=config) diff --git a/data_science/SMSModel/dataset_splitting/splitter.py b/data_science/SMSModel/dataset_splitting/splitter.py new file mode 100644 index 0000000..2edfd8c --- /dev/null +++ b/data_science/SMSModel/dataset_splitting/splitter.py @@ -0,0 +1,182 @@ +"""그룹 보존과 클래스 비율 최적화를 적용한 데이터 분할.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +from sklearn.model_selection import GroupShuffleSplit + +from .config import DatasetSplitConfig + + +@dataclass(frozen=True) +class DatasetSplits: + """분할된 train, validation, test DataFrame 묶음.""" + + train: pd.DataFrame + validation: pd.DataFrame + test: pd.DataFrame + + +def _label_distribution( + df: pd.DataFrame, + *, + label_column: str, + labels: list[str], +) -> np.ndarray: + if df.empty: + return np.zeros(len(labels), dtype=float) + proportions = df[label_column].value_counts(normalize=True) + return np.asarray( + [proportions.get(label, 0.0) for label in labels], + dtype=float, + ) + + +def _candidate_score( + *, + selected: pd.DataFrame, + full_data: pd.DataFrame, + target_size: float, + label_column: str, + labels: list[str], +) -> float: + """목표 행 비율과 전체 클래스 비율에 가까울수록 낮은 점수를 줍니다.""" + size_error = abs((len(selected) / len(full_data)) - target_size) + class_error = np.abs( + _label_distribution( + full_data, + label_column=label_column, + labels=labels, + ) + - _label_distribution( + selected, + label_column=label_column, + labels=labels, + ) + ).sum() + return size_error + (class_error * 2.0) + + +def _contains_all_labels( + df: pd.DataFrame, + *, + label_column: str, + labels: set[str], +) -> bool: + return set(df[label_column].unique()) == labels + + +def _select_best_group_split( + df: pd.DataFrame, + *, + selected_size: float, + config: DatasetSplitConfig, + random_state_offset: int, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """여러 결정적 후보 중 크기와 클래스 비율이 가장 좋은 분할을 고릅니다.""" + labels = sorted(df[config.label_column].unique()) + required_labels = set(labels) + best: tuple[pd.DataFrame, pd.DataFrame] | None = None + best_score = float("inf") + + for candidate_index in range(config.candidate_count): + splitter = GroupShuffleSplit( + n_splits=1, + test_size=selected_size, + random_state=( + config.random_state + random_state_offset + candidate_index + ), + ) + remaining_indices, selected_indices = next( + splitter.split( + df, + y=df[config.label_column], + groups=df[config.group_column], + ) + ) + remaining = df.iloc[remaining_indices] + selected = df.iloc[selected_indices] + + if not _contains_all_labels( + remaining, + label_column=config.label_column, + labels=required_labels, + ) or not _contains_all_labels( + selected, + label_column=config.label_column, + labels=required_labels, + ): + continue + + score = _candidate_score( + selected=selected, + full_data=df, + target_size=selected_size, + label_column=config.label_column, + labels=labels, + ) + if score < best_score: + best_score = score + best = (remaining, selected) + + if best is None: + raise RuntimeError( + "Could not create a grouped split containing every label. " + "Inspect template groups and class distribution." + ) + + return tuple(part.reset_index(drop=True) for part in best) + + +def split_grouped_dataset( + df: pd.DataFrame, + *, + config: DatasetSplitConfig | None = None, +) -> DatasetSplits: + """동일 template_group_id를 보존하며 70/15/15에 가깝게 분할합니다.""" + config = config or DatasetSplitConfig() + required = { + config.group_column, + config.label_column, + config.fingerprint_column, + } + missing = required - set(df.columns) + if missing: + raise ValueError(f"missing split columns: {missing}") + if df.empty: + raise ValueError("cannot split an empty dataset") + if df[list(required)].isna().any().any(): + raise ValueError("split key columns contain missing values") + if df[config.fingerprint_column].duplicated().any(): + raise ValueError("text_fingerprint must be unique before splitting") + if df[config.group_column].nunique() < 3: + raise ValueError("at least three template groups are required") + + train_validation, test = _select_best_group_split( + df, + selected_size=config.test_size, + config=config, + random_state_offset=0, + ) + relative_validation_size = config.val_size / ( + config.train_size + config.val_size + ) + train, validation = _select_best_group_split( + train_validation, + selected_size=relative_validation_size, + config=config, + random_state_offset=100_000, + ) + + parts = {"train": train, "validation": validation, "test": test} + for split_name, part in parts.items(): + part["split"] = split_name + + return DatasetSplits( + train=train.reset_index(drop=True), + validation=validation.reset_index(drop=True), + test=test.reset_index(drop=True), + ) diff --git a/data_science/SMSModel/dataset_splitting/validation.py b/data_science/SMSModel/dataset_splitting/validation.py new file mode 100644 index 0000000..f8f0ab1 --- /dev/null +++ b/data_science/SMSModel/dataset_splitting/validation.py @@ -0,0 +1,84 @@ +"""데이터 split의 누수 및 무결성 검증.""" + +from __future__ import annotations + +import pandas as pd + +from .config import DatasetSplitConfig +from .splitter import DatasetSplits + + +def _named_parts(splits: DatasetSplits): + return ( + ("train", splits.train), + ("validation", splits.validation), + ("test", splits.test), + ) + + +def _validate_pairwise_disjoint( + splits: DatasetSplits, + *, + column: str, + description: str, +) -> None: + parts = _named_parts(splits) + for left_index, (left_name, left) in enumerate(parts): + for right_name, right in parts[left_index + 1 :]: + overlap = set(left[column].astype(str)) & set(right[column].astype(str)) + if overlap: + raise ValueError( + f"{description} leakage between {left_name} and " + f"{right_name}: {sorted(overlap)[:10]}" + ) + + +def validate_dataset_splits( + source: pd.DataFrame, + splits: DatasetSplits, + *, + config: DatasetSplitConfig | None = None, +) -> None: + """그룹·fingerprint 누수, 행 커버리지와 split 명칭을 모두 검증합니다.""" + config = config or DatasetSplitConfig() + if source[config.fingerprint_column].duplicated().any(): + raise ValueError("source contains duplicate text fingerprints") + + for expected_name, part in _named_parts(splits): + if part.empty: + raise ValueError(f"{expected_name} split is empty") + if "split" not in part.columns: + raise ValueError(f"{expected_name} split column is missing") + if set(part["split"].astype(str)) != {expected_name}: + raise ValueError(f"invalid split name in {expected_name} data") + if set(part[config.label_column].unique()) != set( + source[config.label_column].unique() + ): + raise ValueError(f"{expected_name} does not contain every label") + + _validate_pairwise_disjoint( + splits, + column=config.group_column, + description="template group", + ) + _validate_pairwise_disjoint( + splits, + column=config.fingerprint_column, + description="fingerprint", + ) + + combined = pd.concat( + [splits.train, splits.validation, splits.test], + ignore_index=True, + ) + if len(combined) != len(source): + raise ValueError( + f"split row count mismatch: source={len(source)}, " + f"split={len(combined)}" + ) + if combined[config.fingerprint_column].duplicated().any(): + raise ValueError("a fingerprint appears more than once across splits") + if set(source[config.fingerprint_column].astype(str)) != set( + combined[config.fingerprint_column].astype(str) + ): + raise ValueError("split fingerprint coverage mismatch") diff --git a/data_science/SMSModel/evaluation/__init__.py b/data_science/SMSModel/evaluation/__init__.py new file mode 100644 index 0000000..3dc51fc --- /dev/null +++ b/data_science/SMSModel/evaluation/__init__.py @@ -0,0 +1,33 @@ +"""SMS 모델 공통 평가 API""" + +from .evaluator import ( + ModelEvaluationResult, + train_and_evaluate_model, +) +from .latency import ( + LatencyMetrics, + measure_single_inference_latency, +) +from .metrics import ( + ClassificationMetrics, + calculate_classification_metrics, +) +from .reporting import ( + save_model_evaluation_reports, +) +from .threshold import ( + ThresholdSelection, + select_validation_threshold, +) + +__all__ = [ + "ClassificationMetrics", + "LatencyMetrics", + "ModelEvaluationResult", + "ThresholdSelection", + "calculate_classification_metrics", + "measure_single_inference_latency", + "save_model_evaluation_reports", + "select_validation_threshold", + "train_and_evaluate_model", +] \ No newline at end of file diff --git a/data_science/SMSModel/evaluation/evaluator.py b/data_science/SMSModel/evaluation/evaluator.py new file mode 100644 index 0000000..0c3307d --- /dev/null +++ b/data_science/SMSModel/evaluation/evaluator.py @@ -0,0 +1,117 @@ +"""학습, validation threshold 선택, test 평가 흐름""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + +import pandas as pd + +from data_science.SMSModel.evaluation.latency import ( + LatencyMetrics, + measure_single_inference_latency, +) +from data_science.SMSModel.evaluation.metrics import ( + ClassificationMetrics, + calculate_classification_metrics, +) +from data_science.SMSModel.evaluation.threshold import ( + ThresholdSelection, + select_validation_threshold, +) +from data_science.SMSModel.modeling import ( + BasePhishingClassifier, +) + + +@dataclass(frozen=True) +class ModelEvaluationResult: + """단일 모델의 validation 선택 및 test 평가 결과""" + + model_name: str + score_type: str + selected_threshold: float + validation: ThresholdSelection + test_metrics: ClassificationMetrics + latency: LatencyMetrics + metadata: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + + """평과 결과를 딕셔너리 형태로 직렬화""" + return { + "model_name": self.model_name, + "score_type": self.score_type, + "selected_threshold": ( + self.selected_threshold + ), + "validation": asdict(self.validation), + "test_metrics": self.test_metrics.to_dict(), + "latency": asdict(self.latency), + "metadata": self.metadata, + } + + +def train_and_evaluate_model( + model: BasePhishingClassifier, + *, + train_df: pd.DataFrame, + validation_df: pd.DataFrame, + test_df: pd.DataFrame, + target_recall: float = 0.96, + latency_sample_count: int = 100, +) -> ModelEvaluationResult: + + """단일 모델의 공통 학습 및 평가 흐름을 실행""" + + # 학습 데이터셋으로 모델 학습 + model.fit(train_df) + + # Validation 데이터셋에서 목표 Recall을 달성하는 최적 임계값 + validation_scores = model.predict_scores( + validation_df + ) + + threshold_selection = select_validation_threshold( + validation_df["label"].to_numpy(), + validation_scores.values, + target_recall=target_recall, + ) + + # validation에서 확정된 threshold를 변경하지 않고 test에 적용 + test_scores = model.predict_scores(test_df) + + if test_scores.score_type != validation_scores.score_type: + raise ValueError( + "model score type changed between validation and test" + ) + + # 이미 계산한 test score를 재사용 + test_predictions = model.labels_from_scores( + test_scores.values, + threshold=threshold_selection.threshold, + ) + + test_metrics = calculate_classification_metrics( + test_df["label"].to_numpy(), + test_predictions, + ) + + # Test 데이터 기반 실시간 단건 추론 지연시간 측정 + latency = measure_single_inference_latency( + model, + test_df, + sample_count=latency_sample_count, + ) + + # 최종 종합 평가 결과 반환 + return ModelEvaluationResult( + model_name=model.model_name, + score_type=test_scores.score_type.value, + selected_threshold=( + threshold_selection.threshold + ), + validation=threshold_selection, + test_metrics=test_metrics, + latency=latency, + metadata=model.get_metadata(), + ) \ No newline at end of file diff --git a/data_science/SMSModel/evaluation/latency.py b/data_science/SMSModel/evaluation/latency.py new file mode 100644 index 0000000..365b86b --- /dev/null +++ b/data_science/SMSModel/evaluation/latency.py @@ -0,0 +1,100 @@ +"""모델별 단건 추론 시간 측정""" +from __future__ import annotations + +from dataclasses import dataclass +from time import perf_counter_ns + +import numpy as np +import pandas as pd + +from data_science.SMSModel.modeling import ( + BasePhishingClassifier, +) + + +@dataclass(frozen=True) +class LatencyMetrics: + """밀리초 단위 단건 추론 지연시간 통계""" + + sample_count: int + warmup_count: int + average_ms: float + median_ms: float + p95_ms: float + minimum_ms: float + maximum_ms: float + + +def measure_single_inference_latency( + model: BasePhishingClassifier, + df: pd.DataFrame, + *, + sample_count: int = 100, + warmup_count: int = 5, +) -> LatencyMetrics: + + """DataFrame에서 일부 샘플을 선택해 한 건씩 end-to-end 추론 시간 측정""" + if df.empty: + raise ValueError( + "cannot measure latency from an empty DataFrame" + ) + + if sample_count <= 0: + raise ValueError( + "sample_count must be greater than 0" + ) + + if warmup_count < 0: + raise ValueError( + "warmup_count must not be negative" + ) + + measured_count = min(sample_count, len(df)) + + # 항상 같은 샘플을 사용하도록 앞에서부터 선택 + samples = df.iloc[:measured_count] + + # 최초 호출의 lazy initialization과 캐시 영향을 측정에서 제외 + for warmup_index in range(warmup_count): + sample = samples.iloc[ + warmup_index % measured_count + : (warmup_index % measured_count) + 1 + ] + model.predict_scores(sample) + + durations_ms: list[float] = [] + + for row_index in range(measured_count): + single_row = samples.iloc[ + row_index : row_index + 1 + ] + + started_at = perf_counter_ns() + score_output = model.predict_scores(single_row) + finished_at = perf_counter_ns() + + if len(score_output.values) != 1: + raise ValueError( + "single-row inference must return one score" + ) + + durations_ms.append( + (finished_at - started_at) / 1_000_000 + ) + + duration_array = np.asarray( + durations_ms, + dtype=float, + ) + + return LatencyMetrics( + sample_count=measured_count, + warmup_count=warmup_count, + average_ms=float(duration_array.mean()), + median_ms=float(np.median(duration_array)), + p95_ms=float( + np.percentile(duration_array, 95) + ), + minimum_ms=float(duration_array.min()), + maximum_ms=float(duration_array.max()), + ) \ No newline at end of file diff --git a/data_science/SMSModel/evaluation/metrics.py b/data_science/SMSModel/evaluation/metrics.py new file mode 100644 index 0000000..7385ec3 --- /dev/null +++ b/data_science/SMSModel/evaluation/metrics.py @@ -0,0 +1,130 @@ +"""SMS 피싱 모델 공통 평가 지표""" +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + +import numpy as np +from sklearn.metrics import ( + confusion_matrix, + f1_score, + fbeta_score, + precision_score, + recall_score, +) + + +LABEL_ORDER = ["normal", "phishing"] + + +@dataclass(frozen=True) +class ClassificationMetrics: + """피싱 클래스 중심 분류 평가 결과""" + + sample_count: int + precision: float + recall: float + f1: float + f2: float + true_negative: int + false_positive: int + false_negative: int + true_positive: int + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def calculate_classification_metrics( + y_true, + y_pred, +) -> ClassificationMetrics: + + """Precision, Recall, F1, F2와 혼동행렬을 계산""" + + # 입력 데이터를 문자열 형태의 1차워 Numpy 배열로 표준화 + y_true_array = np.asarray(y_true, dtype=str) + y_pred_array = np.asarray(y_pred, dtype=str) + + # 입력 데이터 유효성 검증 + if y_true_array.ndim != 1 or y_pred_array.ndim != 1: + raise ValueError( + "y_true and y_pred must be one-dimensional" + ) + + if len(y_true_array) != len(y_pred_array): + raise ValueError( + "y_true and y_pred must have the same length" + ) + + if len(y_true_array) == 0: + raise ValueError( + "cannot evaluate empty inputs" + ) + + # 라벨 도메인 검증 + allowed_labels = set(LABEL_ORDER) + + if not set(y_true_array).issubset(allowed_labels): + raise ValueError( + "y_true contains unsupported labels" + ) + + if not set(y_pred_array).issubset(allowed_labels): + raise ValueError( + "y_pred contains unsupported labels" + ) + + # 혼동 행렬 + matrix = confusion_matrix( + y_true_array, + y_pred_array, + labels=LABEL_ORDER, + ) + + true_negative = int(matrix[0, 0]) + false_positive = int(matrix[0, 1]) + false_negative = int(matrix[1, 0]) + true_positive = int(matrix[1, 1]) + + # 주요 분류 평가 지표 산출 + return ClassificationMetrics( + sample_count=len(y_true_array), + precision=float( + precision_score( + y_true_array, + y_pred_array, + pos_label="phishing", + zero_division=0, + ) + ), + recall=float( + recall_score( + y_true_array, + y_pred_array, + pos_label="phishing", + zero_division=0, + ) + ), + f1=float( + f1_score( + y_true_array, + y_pred_array, + pos_label="phishing", + zero_division=0, + ) + ), + f2=float( + fbeta_score( + y_true_array, + y_pred_array, + beta=2, + pos_label="phishing", + zero_division=0, + ) + ), + true_negative=true_negative, + false_positive=false_positive, + false_negative=false_negative, + true_positive=true_positive, + ) \ No newline at end of file diff --git a/data_science/SMSModel/evaluation/reporting.py b/data_science/SMSModel/evaluation/reporting.py new file mode 100644 index 0000000..7afbaa8 --- /dev/null +++ b/data_science/SMSModel/evaluation/reporting.py @@ -0,0 +1,187 @@ +"""모델 평가 결과 JSON, CSV, Markdown 출력""" +from __future__ import annotations + +import csv +import json +from pathlib import Path + +from .evaluator import ModelEvaluationResult + + +def save_model_evaluation_json( + results: list[ModelEvaluationResult], + path: Path, +) -> None: + """상세 평가 결과를 JSON으로 저장""" + path.parent.mkdir( + parents=True, + exist_ok=True, + ) + + payload = { + "schema_version": 1, + "models": [ + result.to_dict() + for result in results + ], + } + + # 한글 깨짐 방지 + path.write_text( + json.dumps( + payload, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + +def save_model_evaluation_csv( + results: list[ModelEvaluationResult], + path: Path, +) -> None: + """모델별 핵심 성능 비교표를 CSV로 저장""" + path.parent.mkdir( + parents=True, + exist_ok=True, + ) + + # CSV 컬럼 레이아웃 정의 + fieldnames = [ + "model_name", + "score_type", + "threshold", + "precision", + "recall", + "f1", + "f2", + "false_positive", + "false_negative", + "true_positive", + "true_negative", + "average_inference_ms", + "p95_inference_ms", + ] + + with path.open( + "w", + encoding="utf-8", + newline="", + ) as csv_file: + writer = csv.DictWriter( + csv_file, + fieldnames=fieldnames, + ) + writer.writeheader() + + # 각 모델의 핵심 지표만 추출하여 행 단위로 기록 + for result in results: + metrics = result.test_metrics + + writer.writerow( + { + "model_name": result.model_name, + "score_type": result.score_type, + "threshold": ( + result.selected_threshold + ), + "precision": metrics.precision, + "recall": metrics.recall, + "f1": metrics.f1, + "f2": metrics.f2, + "false_positive": ( + metrics.false_positive + ), + "false_negative": ( + metrics.false_negative + ), + "true_positive": ( + metrics.true_positive + ), + "true_negative": ( + metrics.true_negative + ), + "average_inference_ms": ( + result.latency.average_ms + ), + "p95_inference_ms": ( + result.latency.p95_ms + ), + } + ) + + +def render_model_evaluation_markdown( + results: list[ModelEvaluationResult], +) -> str: + """모델 비교 결과를 Markdown 표로 렌더링""" + lines = [ + "# Phishing Model Evaluation", + "", + "| Model | Threshold | Precision | Recall | F1 | F2 | FN | FP | Avg ms | P95 ms |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + + for result in results: + metrics = result.test_metrics + latency = result.latency + + # 소수점 자릿수 정렬 + lines.append( + f"| {result.model_name} " + f"| {result.selected_threshold:.6f} " + f"| {metrics.precision:.4f} " + f"| {metrics.recall:.4f} " + f"| {metrics.f1:.4f} " + f"| {metrics.f2:.4f} " + f"| {metrics.false_negative} " + f"| {metrics.false_positive} " + f"| {latency.average_ms:.3f} " + f"| {latency.p95_ms:.3f} |" + ) + + return "\n".join(lines) + "\n" + + +def save_model_evaluation_markdown( + results: list[ModelEvaluationResult], + path: Path, +) -> None: + """Markdown 모델 비교표를 저장""" + path.parent.mkdir( + parents=True, + exist_ok=True, + ) + + path.write_text( + render_model_evaluation_markdown(results), + encoding="utf-8", + ) + + +def save_model_evaluation_reports( + results: list[ModelEvaluationResult], + *, + output_directory: Path, +) -> None: + """JSON, CSV, Markdown 결과를 모두 저장""" + if not results: + raise ValueError( + "at least one evaluation result is required" + ) + + save_model_evaluation_json( + results, + output_directory / "model_evaluation.json", + ) + save_model_evaluation_csv( + results, + output_directory / "model_evaluation.csv", + ) + save_model_evaluation_markdown( + results, + output_directory / "model_evaluation.md", + ) \ No newline at end of file diff --git a/data_science/SMSModel/evaluation/threshold.py b/data_science/SMSModel/evaluation/threshold.py new file mode 100644 index 0000000..7b92d35 --- /dev/null +++ b/data_science/SMSModel/evaluation/threshold.py @@ -0,0 +1,170 @@ +"""validation 데이터 기반 threshold 선택""" +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +from sklearn.metrics import ( + fbeta_score, + precision_score, + recall_score, +) + + +@dataclass(frozen=True) +class ThresholdSelection: + """Validation에서 선택된 threshold와 관련 지표""" + + threshold: float + precision: float + recall: float + f2: float + false_negative_count: int + target_recall: float + target_recall_met: bool + + +def _validate_binary_inputs( + y_true: np.ndarray, + scores: np.ndarray, +) -> None: + + """label과 score 입력의 기본 무결성을 검사""" + if y_true.ndim != 1 or scores.ndim != 1: + raise ValueError( + "y_true and scores must be one-dimensional" + ) + + if len(y_true) != len(scores): + raise ValueError( + "y_true and scores must have the same length" + ) + + if len(y_true) == 0: + raise ValueError( + "cannot select threshold from empty inputs" + ) + + if not np.isfinite(scores).all(): + raise ValueError( + "scores must contain only finite numbers" + ) + + allowed_labels = {"normal", "phishing"} + + if not set(y_true).issubset(allowed_labels): + raise ValueError( + f"unsupported labels: {set(y_true) - allowed_labels}" + ) + + +def _candidate_thresholds( + scores: np.ndarray, +) -> np.ndarray: + """validation score에서 결정적인 threshold 후보 생성""" + unique_scores = np.unique(scores) + + lower_boundary = np.nextafter( + unique_scores.min(), + -np.inf, + ) + + return np.concatenate( + [ + np.asarray([lower_boundary]), + unique_scores, + ] + ) + + +def select_validation_threshold( + y_true, + scores, + *, + target_recall: float = 0.96, +) -> ThresholdSelection: + + """Validation set에서 Recall 목표를 우선하면서 F2가 가장 높은 threshold를 선택""" + y_true_array = np.asarray(y_true, dtype=str) + score_array = np.asarray(scores, dtype=float) + + _validate_binary_inputs( + y_true_array, + score_array, + ) + + if not 0.0 < target_recall <= 1.0: + raise ValueError( + "target_recall must be greater than 0 and at most 1" + ) + + candidates: list[ThresholdSelection] = [] + + for threshold in _candidate_thresholds(score_array): + predictions = np.where( + score_array >= threshold, + "phishing", + "normal", + ) + + precision = precision_score( + y_true_array, + predictions, + pos_label="phishing", + zero_division=0, + ) + recall = recall_score( + y_true_array, + predictions, + pos_label="phishing", + zero_division=0, + ) + f2 = fbeta_score( + y_true_array, + predictions, + beta=2, + pos_label="phishing", + zero_division=0, + ) + + false_negative_count = int( + ( + (y_true_array == "phishing") + & (predictions == "normal") + ).sum() + ) + + candidates.append( + ThresholdSelection( + threshold=float(threshold), + precision=float(precision), + recall=float(recall), + f2=float(f2), + false_negative_count=false_negative_count, + target_recall=target_recall, + target_recall_met=bool( + recall >= target_recall + ), + ) + ) + + recall_candidates = [ + candidate + for candidate in candidates + if candidate.target_recall_met + ] + + selection_pool = ( + recall_candidates + if recall_candidates + else candidates + ) + + return max( + selection_pool, + key=lambda candidate: ( + candidate.f2, + candidate.precision, + candidate.threshold, + ), + ) \ No newline at end of file diff --git a/data_science/SMSModel/modeling/__init__.py b/data_science/SMSModel/modeling/__init__.py new file mode 100644 index 0000000..e117e7a --- /dev/null +++ b/data_science/SMSModel/modeling/__init__.py @@ -0,0 +1,21 @@ +"""SMS 피싱 모델 공통 API""" +from .base import ( + BasePhishingClassifier, + ScoreOutput, + ScoreType, +) +from .naive_bayes import ( + NaiveBayesPhishingClassifier, +) +from .artifacts import ( + save_operational_naive_bayes_artifacts, +) + +__all__ = [ + "BasePhishingClassifier", + "NaiveBayesPhishingClassifier", + "ScoreOutput", + "ScoreType", + "save_operational_naive_bayes_artifacts", +] + diff --git a/data_science/SMSModel/modeling/artifacts.py b/data_science/SMSModel/modeling/artifacts.py new file mode 100644 index 0000000..0bb779c --- /dev/null +++ b/data_science/SMSModel/modeling/artifacts.py @@ -0,0 +1,66 @@ +"""모델 Adapter에서 기존 운영 API용 artifact를 저장하는 기능""" +from __future__ import annotations + +from pathlib import Path + +import joblib + +from .naive_bayes import ( + NaiveBayesPhishingClassifier, +) + + +def save_operational_naive_bayes_artifacts( + classifier: NaiveBayesPhishingClassifier, + *, + threshold: float, + model_path: Path, + vectorizer_path: Path, +) -> None: + + """structural NB 모델을 기존 운영 artifact 형식으로 저장""" + + classifier._require_fitted() + + if not classifier.include_structural_features: + raise ValueError( + "only the structural Naive Bayes model " + "can be saved as the operational artifact" + ) + + if not 0.0 <= threshold <= 1.0: + raise ValueError( + "probability threshold must be between 0 and 1" + ) + + model_path = Path(model_path) + vectorizer_path = Path( + vectorizer_path + ) + + model_path.parent.mkdir( + parents=True, + exist_ok=True, + ) + vectorizer_path.parent.mkdir( + parents=True, + exist_ok=True, + ) + + # 기존 API와 완전히 같은 key만 저장 + joblib.dump( + { + "model": classifier.model, + "threshold": float(threshold), + "classes": list( + classifier.classes_ + ), + }, + model_path, + ) + + # 기존 API는 vectorizer를 별도 pkl로 읽음 + joblib.dump( + classifier.vectorizer, + vectorizer_path, + ) \ No newline at end of file diff --git a/data_science/SMSModel/modeling/base.py b/data_science/SMSModel/modeling/base.py new file mode 100644 index 0000000..9ed6ae9 --- /dev/null +++ b/data_science/SMSModel/modeling/base.py @@ -0,0 +1,106 @@ +"""SMS 피싱 분류 모델의 공통 인터페이스.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum +from typing import Any + +import numpy as np +import pandas as pd + + +class ScoreType(str, Enum): + """모델이 반환하는 피싱 점수의 의미.""" + + PROBABILITY = "probability" + DECISION = "decision" + + +@dataclass(frozen=True) +class ScoreOutput: + """값이 클수록 피싱 가능성이 높은 1차원 점수 배열.""" + + values: np.ndarray + score_type: ScoreType + + def __post_init__(self) -> None: + values = np.asarray(self.values) + if values.ndim != 1: + raise ValueError("score values must be one-dimensional") + if not np.isfinite(values).all(): + raise ValueError("score values must contain only finite numbers") + if self.score_type == ScoreType.PROBABILITY and ( + (values < 0.0) | (values > 1.0) + ).any(): + raise ValueError("probability scores must be between 0 and 1") + + +class BasePhishingClassifier(ABC): + """모든 SMS 피싱 모델 Adapter가 구현해야 하는 공통 계약.""" + + @property + @abstractmethod + def model_name(self) -> str: + """보고서와 artifact에서 사용할 모델 식별자.""" + + @property + @abstractmethod + def score_type(self) -> ScoreType: + """모델이 probability 또는 decision 중 무엇을 반환하는지 표시.""" + + @property + def default_threshold(self) -> float: + """점수 유형에 맞는 기본 threshold.""" + return 0.5 if self.score_type == ScoreType.PROBABILITY else 0.0 + + @abstractmethod + def fit(self, train_df: pd.DataFrame) -> BasePhishingClassifier: + """train 데이터만 사용해 모델을 학습합니다.""" + + @abstractmethod + def predict_scores(self, df: pd.DataFrame) -> ScoreOutput: + """각 행의 피싱 점수를 반환합니다.""" + + @staticmethod + def labels_from_scores( + scores: np.ndarray, + *, + threshold: float, + ) -> np.ndarray: + """이미 계산한 score를 label로 변환해 test 중복 추론을 피합니다.""" + score_array = np.asarray(scores, dtype=float) + if score_array.ndim != 1: + raise ValueError("scores must be one-dimensional") + if not np.isfinite(score_array).all(): + raise ValueError("scores must contain only finite numbers") + return np.where( + score_array >= threshold, + "phishing", + "normal", + ) + + def predict( + self, + df: pd.DataFrame, + *, + threshold: float | None = None, + ) -> np.ndarray: + """기본 또는 지정 threshold를 적용해 label을 반환합니다.""" + selected_threshold = ( + self.default_threshold if threshold is None else threshold + ) + scores = self.predict_scores(df) + return self.labels_from_scores( + scores.values, + threshold=selected_threshold, + ) + + def get_metadata(self) -> dict[str, Any]: + """artifact와 보고서에 기록할 공통 metadata.""" + return { + "model_name": self.model_name, + "score_type": self.score_type.value, + "default_threshold": self.default_threshold, + } diff --git a/data_science/SMSModel/modeling/naive_bayes.py b/data_science/SMSModel/modeling/naive_bayes.py new file mode 100644 index 0000000..199672c --- /dev/null +++ b/data_science/SMSModel/modeling/naive_bayes.py @@ -0,0 +1,323 @@ +"""기존 SafeFam Naive Bayes 모델의 공통 평가기 Adapter""" +from __future__ import annotations + +from typing import Any + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack, spmatrix +from sklearn.calibration import CalibratedClassifierCV +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.naive_bayes import ComplementNB + +from app.analysis.text.preprocessing import ( + extract_struct_feature_matrix, +) +from data_science.SMSModel.modeling.base import ( + BasePhishingClassifier, + ScoreOutput, + ScoreType, +) + + +# 기존 운영 NB 설정을 그대로 유지 +DEFAULT_NB_ALPHA = 0.01 +DEFAULT_CALIBRATION_METHOD = "isotonic" +DEFAULT_CALIBRATION_CV = 5 + +DEFAULT_NGRAM_RANGE = (2, 4) +DEFAULT_MIN_DF = 2 +DEFAULT_MAX_DF = 0.95 +DEFAULT_MAX_FEATURES = 8_000 + + +class NaiveBayesPhishingClassifier( + BasePhishingClassifier +): + """Calibrated ComplementNB 기반 피싱 분류기 Adapter""" + + def __init__( + self, + *, + include_structural_features: bool, + alpha: float = DEFAULT_NB_ALPHA, + calibration_method: str = DEFAULT_CALIBRATION_METHOD, + calibration_cv: int = DEFAULT_CALIBRATION_CV, + ) -> None: + if alpha <= 0: + raise ValueError( + "alpha must be greater than 0" + ) + + if calibration_cv < 2: + raise ValueError( + "calibration_cv must be at least 2" + ) + + self.include_structural_features = ( + include_structural_features + ) + self.alpha = alpha + self.calibration_method = ( + calibration_method + ) + self.calibration_cv = calibration_cv + + self.vectorizer = self._build_vectorizer() + self.model: CalibratedClassifierCV | None = None + self.classes_: list[str] | None = None + + @property + def model_name(self) -> str: + + """보고서에서 두 NB 실험을 구분할 이름을 반환""" + + if self.include_structural_features: + return "naive_bayes_structural" + + return "naive_bayes_text_only" + + @property + def score_type(self) -> ScoreType: + + """CalibratedClassifierCV를 사용하므로 0~1 확률 반환""" + + return ScoreType.PROBABILITY + + @staticmethod + def _build_vectorizer() -> CountVectorizer: + + """기존 운영 모델과 동일한 문자 n-gram CountVectorizer 생성""" + + return CountVectorizer( + analyzer="char_wb", + ngram_range=DEFAULT_NGRAM_RANGE, + min_df=DEFAULT_MIN_DF, + max_df=DEFAULT_MAX_DF, + max_features=DEFAULT_MAX_FEATURES, + ) + + @staticmethod + def _validate_dataframe( + df: pd.DataFrame, + *, + require_label: bool, + ) -> None: + + """학습 또는 추론에 필요한 DataFrame 컬럼 검사""" + + required_columns = { + "text", + "text_norm", + "has_url", + } + + if require_label: + required_columns.add("label") + + missing = required_columns - set(df.columns) + + if missing: + raise ValueError( + f"missing Naive Bayes columns: {missing}" + ) + + if df.empty: + raise ValueError( + "Naive Bayes input DataFrame is empty" + ) + + if df[list(required_columns)].isna().any().any(): + raise ValueError( + "Naive Bayes input contains missing values" + ) + + if require_label: + labels = set(df["label"].astype(str)) + + if not labels.issubset( + {"normal", "phishing"} + ): + raise ValueError( + f"unsupported labels: " + f"{labels - {'normal', 'phishing'}}" + ) + + if labels != {"normal", "phishing"}: + raise ValueError( + "training data must contain both " + "normal and phishing labels" + ) + + def _prepare_normalized_text( + self, + df: pd.DataFrame, + ) -> pd.Series: + + """text_norm을 사용하되 공통 전처리와 일치하는지 보장""" + + return df["text_norm"].astype(str) + + def _build_feature_matrix( + self, + df: pd.DataFrame, + *, + fit_vectorizer: bool, + ) -> spmatrix: + + """모델 모드에 맞는 sparse feature matrix 생성""" + + text_norm = self._prepare_normalized_text(df) + + if fit_vectorizer: + text_features = ( + self.vectorizer.fit_transform( + text_norm + ) + ) + else: + text_features = ( + self.vectorizer.transform( + text_norm + ) + ) + + if not self.include_structural_features: + return text_features.tocsr() + + structural_features = ( + extract_struct_feature_matrix( + df["text"], + df["has_url"], + ) + ) + + return hstack( + [ + text_features, + csr_matrix(structural_features), + ], + format="csr", + ) + + def fit( + self, + train_df: pd.DataFrame, + ) -> BasePhishingClassifier: + + """train split만 사용해 ComplementNB와 확률 보정기 학습""" + + self._validate_dataframe( + train_df, + require_label=True, + ) + + feature_matrix = self._build_feature_matrix( + train_df, + fit_vectorizer=True, + ) + + base_model = ComplementNB( + alpha=self.alpha, + ) + + calibrated_model = CalibratedClassifierCV( + estimator=base_model, + method=self.calibration_method, + cv=self.calibration_cv, + ) + + calibrated_model.fit( + feature_matrix, + train_df["label"].astype(str), + ) + + self.model = calibrated_model + self.classes_ = list( + calibrated_model.classes_ + ) + + if "phishing" not in self.classes_: + raise RuntimeError( + "trained model does not contain phishing class" + ) + + return self + + def _require_fitted(self) -> None: + + """학습되지 않은 Adapter의 추론 차단""" + + if self.model is None or self.classes_ is None: + raise RuntimeError( + "Naive Bayes model is not fitted" + ) + + def predict_scores( + self, + df: pd.DataFrame, + ) -> ScoreOutput: + + """각 메시지의 보정된 phishing 확률 반환""" + + self._require_fitted() + + self._validate_dataframe( + df, + require_label=False, + ) + + feature_matrix = self._build_feature_matrix( + df, + fit_vectorizer=False, + ) + + phishing_index = self.classes_.index( + "phishing" + ) + + probabilities = self.model.predict_proba( + feature_matrix + )[:, phishing_index] + + return ScoreOutput( + values=np.asarray( + probabilities, + dtype=float, + ), + score_type=self.score_type, + ) + + def get_metadata(self) -> dict[str, Any]: + + """평가 보고서와 artifact 추적에 필요한 NB 설정을 반환""" + + return { + **super().get_metadata(), + "classifier": "ComplementNB", + "alpha": self.alpha, + "calibration_method": ( + self.calibration_method + ), + "calibration_cv": self.calibration_cv, + "include_structural_features": ( + self.include_structural_features + ), + "vectorizer": { + "type": "CountVectorizer", + "analyzer": "char_wb", + "ngram_range": list( + DEFAULT_NGRAM_RANGE + ), + "min_df": DEFAULT_MIN_DF, + "max_df": DEFAULT_MAX_DF, + "max_features": ( + DEFAULT_MAX_FEATURES + ), + }, + "structural_feature_count": ( + 6 + if self.include_structural_features + else 0 + ), + } diff --git a/data_science/SMSModel/reporting/__init__.py b/data_science/SMSModel/reporting/__init__.py new file mode 100644 index 0000000..c340033 --- /dev/null +++ b/data_science/SMSModel/reporting/__init__.py @@ -0,0 +1,16 @@ +"""SMS 학습 및 데이터 통계 보고서 공개 API""" +from .dataset_split_report import ( + build_dataset_split_summary, + calculate_dataset_fingerprint, + generate_dataset_split_reports, + render_dataset_split_markdown, + save_dataset_split_reports, +) + +__all__ = [ + "build_dataset_split_summary", + "calculate_dataset_fingerprint", + "generate_dataset_split_reports", + "render_dataset_split_markdown", + "save_dataset_split_reports", +] diff --git a/data_science/SMSModel/reporting/dataset_split_report.py b/data_science/SMSModel/reporting/dataset_split_report.py new file mode 100644 index 0000000..550b549 --- /dev/null +++ b/data_science/SMSModel/reporting/dataset_split_report.py @@ -0,0 +1,444 @@ +""" SMS 데이터 분할 검증 및 통계 보고서 생성""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import pandas as pd + +from data_science.SMSModel.dataset_splitting import ( + DatasetSplitConfig, + DatasetSplits, + validate_dataset_splits, +) +from data_science.SMSModel.template_grouping import ( + TemplateGroupingConfig, +) + + +REPORT_SCHEMA_VERSION = 1 + +REQUIRED_REPORT_COLUMNS = { + "text_fingerprint", + "template_group_id", + "label", + "type", + "split", +} + + +def calculate_dataset_fingerprint( + source: pd.DataFrame, +) -> str: + """전체 학습 후보 데이터셋을 나타내는 SHA-256 fingerprint를 생성""" + required_columns = { + "text_fingerprint", + "template_group_id", + "label", + "type", + } + missing = required_columns - set(source.columns) + + if missing: + raise ValueError( + f"cannot calculate dataset fingerprint; " + f"missing columns: {missing}" + ) + + canonical_rows: list[str] = [] + + sorted_source = source.sort_values( + by=[ + "text_fingerprint", + "template_group_id", + "label", + "type", + ] + ) + + for row in sorted_source[ + [ + "text_fingerprint", + "template_group_id", + "label", + "type", + ] + ].itertuples(index=False, name=None): + # JSON 직렬화를 사용해 문자열 결합 구분자가 실제 데이터와 충돌하는 문제를 방지 + canonical_rows.append( + json.dumps( + [str(value) for value in row], + ensure_ascii=False, + separators=(",", ":"), + ) + ) + + canonical_dataset = "\n".join(canonical_rows) + + return hashlib.sha256( + canonical_dataset.encode("utf-8") + ).hexdigest() + + +def _label_statistics( + df: pd.DataFrame, +) -> dict[str, dict[str, int | float]]: + """normal/phishing별 건수와 비율을 반환""" + result: dict[str, dict[str, int | float]] = {} + + for label in ("normal", "phishing"): + count = int((df["label"] == label).sum()) + + result[label] = { + "count": count, + "ratio": round(count / len(df), 6) if len(df) else 0.0, + } + + return result + + +def _type_statistics( + df: pd.DataFrame, +) -> dict[str, dict[str, int | float]]: + """메시지 type별 건수와 split 내부 비율을 반환""" + counts = df["type"].astype(str).value_counts() + + return { + message_type: { + "count": int(count), + "ratio": round(int(count) / len(df), 6) if len(df) else 0.0, + } + for message_type, count in counts.sort_index().items() + } + + +def _split_statistics( + df: pd.DataFrame, +) -> dict[str, Any]: + """단일 split의 건수, 그룹 수, 클래스 분포와 유형 분포를 계산""" + group_sizes = df["template_group_id"].value_counts() + + return { + "row_count": int(len(df)), + "group_count": int(df["template_group_id"].nunique()), + "largest_group_size": ( + int(group_sizes.max()) + if not group_sizes.empty + else 0 + ), + "labels": _label_statistics(df), + "types": _type_statistics(df), + } + + +def _find_pairwise_overlaps( + splits: DatasetSplits, + *, + column: str, +) -> dict[str, dict[str, Any]]: + """train/validation/test 사이의 그룹 또는 fingerprint 교차를 계산""" + named_splits = { + "train": splits.train, + "validation": splits.validation, + "test": splits.test, + } + + pairs = ( + ("train", "validation"), + ("train", "test"), + ("validation", "test"), + ) + + result: dict[str, dict[str, Any]] = {} + + for left_name, right_name in pairs: + left_values = set( + named_splits[left_name][column].astype(str) + ) + right_values = set( + named_splits[right_name][column].astype(str) + ) + + overlap = sorted(left_values & right_values) + + result[f"{left_name}_vs_{right_name}"] = { + "count": len(overlap), + # 오류가 발생했을 때 보고서나 로그가 지나치게 커지지 않도록 앞의 10개만 예시로 남김 + "examples": overlap[:10], + } + + return result + + +def _total_overlap_count( + overlap_result: dict[str, dict[str, Any]], +) -> int: + """교차 검증 결과의 전체 중복 건수를 계산""" + return sum( + int(pair_result["count"]) + for pair_result in overlap_result.values() + ) + + +def build_dataset_split_summary( + source: pd.DataFrame, + splits: DatasetSplits, + *, + split_config: DatasetSplitConfig, + grouping_config: TemplateGroupingConfig, +) -> dict[str, Any]: + """검증을 수행하고 데이터 분할 통계 보고서 dictionary를 생성""" + + validate_dataset_splits( + source, + splits, + config=split_config, + ) + + group_overlaps = _find_pairwise_overlaps( + splits, + column=split_config.group_column, + ) + fingerprint_overlaps = _find_pairwise_overlaps( + splits, + column=split_config.fingerprint_column, + ) + + total_row_count = len(source) + + return { + "schema_version": REPORT_SCHEMA_VERSION, + "dataset_fingerprint": calculate_dataset_fingerprint( + source + ), + "configuration": { + "template_grouping": { + "similarity_threshold": ( + grouping_config.similarity_threshold + ), + "ngram_range": list( + grouping_config.ngram_range + ), + "min_df": grouping_config.min_df, + "max_features": grouping_config.max_features, + }, + "dataset_split": { + "train_size": split_config.train_size, + "validation_size": split_config.val_size, + "test_size": split_config.test_size, + "random_state": split_config.random_state, + "candidate_count": ( + split_config.candidate_count + ), + }, + }, + "validation": { + "passed": True, + "group_overlap_count": _total_overlap_count( + group_overlaps + ), + "fingerprint_overlap_count": ( + _total_overlap_count( + fingerprint_overlaps + ) + ), + "group_overlaps": group_overlaps, + "fingerprint_overlaps": fingerprint_overlaps, + }, + "dataset": { + "row_count": int(total_row_count), + "group_count": int( + source["template_group_id"].nunique() + ), + "labels": _label_statistics(source), + "types": _type_statistics(source), + }, + "splits": { + "train": { + **_split_statistics(splits.train), + "dataset_ratio": round( + len(splits.train) / total_row_count, + 6, + ), + }, + "validation": { + **_split_statistics(splits.validation), + "dataset_ratio": round( + len(splits.validation) / total_row_count, + 6, + ), + }, + "test": { + **_split_statistics(splits.test), + "dataset_ratio": round( + len(splits.test) / total_row_count, + 6, + ), + }, + }, + } + + +def render_dataset_split_markdown( + summary: dict[str, Any], +) -> str: + """JSON summary를 사람이 검토하기 쉬운 Markdown 문서로 변환""" + lines = [ + "# SMS Dataset Split Summary", + "", + "## Dataset", + "", + f"- Schema version: `{summary['schema_version']}`", + ( + "- Dataset fingerprint: " + f"`{summary['dataset_fingerprint']}`" + ), + ( + "- Total rows: " + f"{summary['dataset']['row_count']}" + ), + ( + "- Template groups: " + f"{summary['dataset']['group_count']}" + ), + "", + "## Configuration", + "", + ( + "- Template similarity threshold: " + f"`{summary['configuration']['template_grouping']['similarity_threshold']}`" + ), + ( + "- Template n-gram range: " + f"`{summary['configuration']['template_grouping']['ngram_range']}`" + ), + ( + "- Random state: " + f"`{summary['configuration']['dataset_split']['random_state']}`" + ), + "", + "## Split Overview", + "", + "| Split | Rows | Ratio | Groups | Normal | Phishing |", + "|---|---:|---:|---:|---:|---:|", + ] + + for split_name in ("train", "validation", "test"): + split = summary["splits"][split_name] + + lines.append( + f"| {split_name} " + f"| {split['row_count']} " + f"| {split['dataset_ratio']:.2%} " + f"| {split['group_count']} " + f"| {split['labels']['normal']['count']} " + f"({split['labels']['normal']['ratio']:.2%}) " + f"| {split['labels']['phishing']['count']} " + f"({split['labels']['phishing']['ratio']:.2%}) |" + ) + + validation = summary["validation"] + + lines.extend( + [ + "", + "## Leakage Validation", + "", + f"- Passed: `{validation['passed']}`", + ( + "- Template group overlap count: " + f"`{validation['group_overlap_count']}`" + ), + ( + "- Fingerprint overlap count: " + f"`{validation['fingerprint_overlap_count']}`" + ), + "", + "## Message Type Distribution", + "", + ] + ) + + for split_name in ("train", "validation", "test"): + lines.extend( + [ + f"### {split_name.title()}", + "", + "| Type | Count | Ratio |", + "|---|---:|---:|", + ] + ) + + for message_type, statistics in ( + summary["splits"][split_name]["types"].items() + ): + # type 값에 |가 포함되면 Markdown table이 깨지므로 escape + escaped_type = message_type.replace("|", "\\|") + + lines.append( + f"| {escaped_type} " + f"| {statistics['count']} " + f"| {statistics['ratio']:.2%} |" + ) + + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + +def save_dataset_split_reports( + summary: dict[str, Any], + *, + json_path: Path, + markdown_path: Path, +) -> None: + """JSON과 Markdown 보고서를 저장""" + json_path.parent.mkdir(parents=True, exist_ok=True) + markdown_path.parent.mkdir( + parents=True, + exist_ok=True, + ) + + json_path.write_text( + json.dumps( + summary, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + markdown_path.write_text( + render_dataset_split_markdown(summary), + encoding="utf-8", + ) + + +def generate_dataset_split_reports( + source: pd.DataFrame, + splits: DatasetSplits, + *, + split_config: DatasetSplitConfig, + grouping_config: TemplateGroupingConfig, + json_path: Path, + markdown_path: Path, +) -> dict[str, Any]: + """검증 → summary 생성 → JSON/Markdown 저장을 한 번에 실행""" + summary = build_dataset_split_summary( + source, + splits, + split_config=split_config, + grouping_config=grouping_config, + ) + + save_dataset_split_reports( + summary, + json_path=json_path, + markdown_path=markdown_path, + ) + + return summary \ No newline at end of file diff --git a/data_science/SMSModel/reports/dataset_split_summary.json b/data_science/SMSModel/reports/dataset_split_summary.json new file mode 100644 index 0000000..cd7a9eb --- /dev/null +++ b/data_science/SMSModel/reports/dataset_split_summary.json @@ -0,0 +1,271 @@ +{ + "configuration": { + "dataset_split": { + "candidate_count": 500, + "random_state": 42, + "test_size": 0.15, + "train_size": 0.7, + "validation_size": 0.15 + }, + "template_grouping": { + "max_features": 50000, + "min_df": 1, + "ngram_range": [ + 2, + 5 + ], + "similarity_threshold": 0.88 + } + }, + "dataset": { + "group_count": 539, + "labels": { + "normal": { + "count": 373, + "ratio": 0.456548 + }, + "phishing": { + "count": 444, + "ratio": 0.543452 + } + }, + "row_count": 817, + "types": { + "경조사사칭": { + "count": 68, + "ratio": 0.083231 + }, + "금융기관사칭": { + "count": 90, + "ratio": 0.110159 + }, + "기타피싱": { + "count": 81, + "ratio": 0.099143 + }, + "이벤트당첨사칭": { + "count": 93, + "ratio": 0.113831 + }, + "일상대화": { + "count": 155, + "ratio": 0.189718 + }, + "정부공공기관사칭": { + "count": 24, + "ratio": 0.029376 + }, + "정상알림톡": { + "count": 218, + "ratio": 0.26683 + }, + "지인사칭": { + "count": 46, + "ratio": 0.056304 + }, + "택배사칭": { + "count": 42, + "ratio": 0.051408 + } + } + }, + "dataset_fingerprint": "1db45d5f2c3d3d17726888b05cd625e0d0a51deef3dc8ab94016a9ee97af18f5", + "schema_version": 1, + "splits": { + "test": { + "dataset_ratio": 0.150551, + "group_count": 81, + "labels": { + "normal": { + "count": 56, + "ratio": 0.455285 + }, + "phishing": { + "count": 67, + "ratio": 0.544715 + } + }, + "largest_group_size": 10, + "row_count": 123, + "types": { + "경조사사칭": { + "count": 8, + "ratio": 0.065041 + }, + "금융기관사칭": { + "count": 17, + "ratio": 0.138211 + }, + "기타피싱": { + "count": 9, + "ratio": 0.073171 + }, + "이벤트당첨사칭": { + "count": 16, + "ratio": 0.130081 + }, + "일상대화": { + "count": 26, + "ratio": 0.211382 + }, + "정부공공기관사칭": { + "count": 5, + "ratio": 0.04065 + }, + "정상알림톡": { + "count": 30, + "ratio": 0.243902 + }, + "지인사칭": { + "count": 2, + "ratio": 0.01626 + }, + "택배사칭": { + "count": 10, + "ratio": 0.081301 + } + } + }, + "train": { + "dataset_ratio": 0.698898, + "group_count": 377, + "labels": { + "normal": { + "count": 261, + "ratio": 0.457093 + }, + "phishing": { + "count": 310, + "ratio": 0.542907 + } + }, + "largest_group_size": 19, + "row_count": 571, + "types": { + "경조사사칭": { + "count": 56, + "ratio": 0.098074 + }, + "금융기관사칭": { + "count": 49, + "ratio": 0.085814 + }, + "기타피싱": { + "count": 50, + "ratio": 0.087566 + }, + "이벤트당첨사칭": { + "count": 72, + "ratio": 0.126095 + }, + "일상대화": { + "count": 103, + "ratio": 0.180385 + }, + "정부공공기관사칭": { + "count": 18, + "ratio": 0.031524 + }, + "정상알림톡": { + "count": 158, + "ratio": 0.276708 + }, + "지인사칭": { + "count": 43, + "ratio": 0.075306 + }, + "택배사칭": { + "count": 22, + "ratio": 0.038529 + } + } + }, + "validation": { + "dataset_ratio": 0.150551, + "group_count": 81, + "labels": { + "normal": { + "count": 56, + "ratio": 0.455285 + }, + "phishing": { + "count": 67, + "ratio": 0.544715 + } + }, + "largest_group_size": 17, + "row_count": 123, + "types": { + "경조사사칭": { + "count": 4, + "ratio": 0.03252 + }, + "금융기관사칭": { + "count": 24, + "ratio": 0.195122 + }, + "기타피싱": { + "count": 22, + "ratio": 0.178862 + }, + "이벤트당첨사칭": { + "count": 5, + "ratio": 0.04065 + }, + "일상대화": { + "count": 26, + "ratio": 0.211382 + }, + "정부공공기관사칭": { + "count": 1, + "ratio": 0.00813 + }, + "정상알림톡": { + "count": 30, + "ratio": 0.243902 + }, + "지인사칭": { + "count": 1, + "ratio": 0.00813 + }, + "택배사칭": { + "count": 10, + "ratio": 0.081301 + } + } + } + }, + "validation": { + "fingerprint_overlap_count": 0, + "fingerprint_overlaps": { + "train_vs_test": { + "count": 0, + "examples": [] + }, + "train_vs_validation": { + "count": 0, + "examples": [] + }, + "validation_vs_test": { + "count": 0, + "examples": [] + } + }, + "group_overlap_count": 0, + "group_overlaps": { + "train_vs_test": { + "count": 0, + "examples": [] + }, + "train_vs_validation": { + "count": 0, + "examples": [] + }, + "validation_vs_test": { + "count": 0, + "examples": [] + } + }, + "passed": true + } +} diff --git a/data_science/SMSModel/reports/dataset_split_summary.md b/data_science/SMSModel/reports/dataset_split_summary.md new file mode 100644 index 0000000..ee1bc01 --- /dev/null +++ b/data_science/SMSModel/reports/dataset_split_summary.md @@ -0,0 +1,72 @@ +# SMS Dataset Split Summary + +## Dataset + +- Schema version: `1` +- Dataset fingerprint: `1db45d5f2c3d3d17726888b05cd625e0d0a51deef3dc8ab94016a9ee97af18f5` +- Total rows: 817 +- Template groups: 539 + +## Configuration + +- Template similarity threshold: `0.88` +- Template n-gram range: `[2, 5]` +- Random state: `42` + +## Split Overview + +| Split | Rows | Ratio | Groups | Normal | Phishing | +|---|---:|---:|---:|---:|---:| +| train | 571 | 69.89% | 377 | 261 (45.71%) | 310 (54.29%) | +| validation | 123 | 15.06% | 81 | 56 (45.53%) | 67 (54.47%) | +| test | 123 | 15.06% | 81 | 56 (45.53%) | 67 (54.47%) | + +## Leakage Validation + +- Passed: `True` +- Template group overlap count: `0` +- Fingerprint overlap count: `0` + +## Message Type Distribution + +### Train + +| Type | Count | Ratio | +|---|---:|---:| +| 경조사사칭 | 56 | 9.81% | +| 금융기관사칭 | 49 | 8.58% | +| 기타피싱 | 50 | 8.76% | +| 이벤트당첨사칭 | 72 | 12.61% | +| 일상대화 | 103 | 18.04% | +| 정부공공기관사칭 | 18 | 3.15% | +| 정상알림톡 | 158 | 27.67% | +| 지인사칭 | 43 | 7.53% | +| 택배사칭 | 22 | 3.85% | + +### Validation + +| Type | Count | Ratio | +|---|---:|---:| +| 경조사사칭 | 4 | 3.25% | +| 금융기관사칭 | 24 | 19.51% | +| 기타피싱 | 22 | 17.89% | +| 이벤트당첨사칭 | 5 | 4.06% | +| 일상대화 | 26 | 21.14% | +| 정부공공기관사칭 | 1 | 0.81% | +| 정상알림톡 | 30 | 24.39% | +| 지인사칭 | 1 | 0.81% | +| 택배사칭 | 10 | 8.13% | + +### Test + +| Type | Count | Ratio | +|---|---:|---:| +| 경조사사칭 | 8 | 6.50% | +| 금융기관사칭 | 17 | 13.82% | +| 기타피싱 | 9 | 7.32% | +| 이벤트당첨사칭 | 16 | 13.01% | +| 일상대화 | 26 | 21.14% | +| 정부공공기관사칭 | 5 | 4.06% | +| 정상알림톡 | 30 | 24.39% | +| 지인사칭 | 2 | 1.63% | +| 택배사칭 | 10 | 8.13% | diff --git a/data_science/SMSModel/feature_scores_full.csv b/data_science/SMSModel/reports/feature_scores_full.csv similarity index 100% rename from data_science/SMSModel/feature_scores_full.csv rename to data_science/SMSModel/reports/feature_scores_full.csv diff --git a/data_science/SMSModel/feature_importance.png b/data_science/SMSModel/reports/figures/feature_importance.png similarity index 100% rename from data_science/SMSModel/feature_importance.png rename to data_science/SMSModel/reports/figures/feature_importance.png diff --git a/data_science/SMSModel/risk_distribution_fig1.png b/data_science/SMSModel/reports/figures/risk_distribution_fig1.png similarity index 100% rename from data_science/SMSModel/risk_distribution_fig1.png rename to data_science/SMSModel/reports/figures/risk_distribution_fig1.png diff --git a/data_science/SMSModel/risk_distribution_fig2.png b/data_science/SMSModel/reports/figures/risk_distribution_fig2.png similarity index 100% rename from data_science/SMSModel/risk_distribution_fig2.png rename to data_science/SMSModel/reports/figures/risk_distribution_fig2.png diff --git a/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.csv b/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.csv new file mode 100644 index 0000000..c20aa68 --- /dev/null +++ b/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.csv @@ -0,0 +1,3 @@ +model_name,score_type,threshold,precision,recall,f1,f2,false_positive,false_negative,true_positive,true_negative,average_inference_ms,p95_inference_ms +naive_bayes_text_only,probability,0.06384636081639411,0.5447154471544715,1.0,0.7052631578947368,0.8567774936061381,56,0,67,0,3.184648,4.11453 +naive_bayes_structural,probability,0.05692328389331719,0.5447154471544715,1.0,0.7052631578947368,0.8567774936061381,56,0,67,0,3.4682509999999995,4.868885 diff --git a/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.json b/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.json new file mode 100644 index 0000000..5452ad9 --- /dev/null +++ b/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.json @@ -0,0 +1,117 @@ +{ + "models": [ + { + "latency": { + "average_ms": 3.184648, + "maximum_ms": 4.7073, + "median_ms": 3.06105, + "minimum_ms": 2.6281, + "p95_ms": 4.11453, + "sample_count": 100, + "warmup_count": 5 + }, + "metadata": { + "alpha": 0.01, + "calibration_cv": 5, + "calibration_method": "isotonic", + "classifier": "ComplementNB", + "default_threshold": 0.5, + "include_structural_features": false, + "model_name": "naive_bayes_text_only", + "score_type": "probability", + "structural_feature_count": 0, + "vectorizer": { + "analyzer": "char_wb", + "max_df": 0.95, + "max_features": 8000, + "min_df": 2, + "ngram_range": [ + 2, + 4 + ], + "type": "CountVectorizer" + } + }, + "model_name": "naive_bayes_text_only", + "score_type": "probability", + "selected_threshold": 0.06384636081639411, + "test_metrics": { + "f1": 0.7052631578947368, + "f2": 0.8567774936061381, + "false_negative": 0, + "false_positive": 56, + "precision": 0.5447154471544715, + "recall": 1.0, + "sample_count": 123, + "true_negative": 0, + "true_positive": 67 + }, + "validation": { + "f2": 0.8567774936061381, + "false_negative_count": 0, + "precision": 0.5447154471544715, + "recall": 1.0, + "target_recall": 0.96, + "target_recall_met": true, + "threshold": 0.06384636081639411 + } + }, + { + "latency": { + "average_ms": 3.4682509999999995, + "maximum_ms": 6.1769, + "median_ms": 3.2692, + "minimum_ms": 2.8544, + "p95_ms": 4.868885, + "sample_count": 100, + "warmup_count": 5 + }, + "metadata": { + "alpha": 0.01, + "calibration_cv": 5, + "calibration_method": "isotonic", + "classifier": "ComplementNB", + "default_threshold": 0.5, + "include_structural_features": true, + "model_name": "naive_bayes_structural", + "score_type": "probability", + "structural_feature_count": 6, + "vectorizer": { + "analyzer": "char_wb", + "max_df": 0.95, + "max_features": 8000, + "min_df": 2, + "ngram_range": [ + 2, + 4 + ], + "type": "CountVectorizer" + } + }, + "model_name": "naive_bayes_structural", + "score_type": "probability", + "selected_threshold": 0.05692328389331719, + "test_metrics": { + "f1": 0.7052631578947368, + "f2": 0.8567774936061381, + "false_negative": 0, + "false_positive": 56, + "precision": 0.5447154471544715, + "recall": 1.0, + "sample_count": 123, + "true_negative": 0, + "true_positive": 67 + }, + "validation": { + "f2": 0.8567774936061381, + "false_negative_count": 0, + "precision": 0.5447154471544715, + "recall": 1.0, + "target_recall": 0.96, + "target_recall_met": true, + "threshold": 0.05692328389331719 + } + } + ], + "schema_version": 1 +} diff --git a/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.md b/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.md new file mode 100644 index 0000000..0ccb887 --- /dev/null +++ b/data_science/SMSModel/reports/model_evaluation/naive_bayes_baseline/model_evaluation.md @@ -0,0 +1,6 @@ +# Phishing Model Evaluation + +| Model | Threshold | Precision | Recall | F1 | F2 | FN | FP | Avg ms | P95 ms | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| naive_bayes_text_only | 0.063846 | 0.5447 | 1.0000 | 0.7053 | 0.8568 | 0 | 56 | 3.185 | 4.115 | +| naive_bayes_structural | 0.056923 | 0.5447 | 1.0000 | 0.7053 | 0.8568 | 0 | 56 | 3.468 | 4.869 | diff --git a/data_science/SMSModel/run_naive_bayes_baseline.py b/data_science/SMSModel/run_naive_bayes_baseline.py new file mode 100644 index 0000000..66d6141 --- /dev/null +++ b/data_science/SMSModel/run_naive_bayes_baseline.py @@ -0,0 +1,152 @@ +"""Naive Bayes text-only 및 structural baseline 실행기""" +from __future__ import annotations + +from pathlib import Path + +from data_science.SMSModel.evaluation import ( + ModelEvaluationResult, + save_model_evaluation_reports, + train_and_evaluate_model, +) +from data_science.SMSModel.modeling import ( + NaiveBayesPhishingClassifier, +) +from data_science.SMSModel.train_sms import ( + DATA_PATH, + MODEL_PATH, + VECTORIZER_PATH, + load_data, + split_data, +) + + +SMS_MODEL_DIR = Path(__file__).resolve().parent + +NB_REPORT_DIRECTORY = ( + SMS_MODEL_DIR + / "reports" + / "model_evaluation" + / "naive_bayes_baseline" +) + +TARGET_PHISHING_RECALL = 0.96 +LATENCY_SAMPLE_COUNT = 100 + + +def evaluate_naive_bayes_baselines() -> list[ + ModelEvaluationResult +]: + """동일한 leakage-safe split에서 두 NB 모델을 평가""" + + dataset, _holdout = load_data( + DATA_PATH + ) + + splits = split_data(dataset) + + text_only_model = ( + NaiveBayesPhishingClassifier( + include_structural_features=False, + ) + ) + + structural_model = ( + NaiveBayesPhishingClassifier( + include_structural_features=True, + ) + ) + + text_only_result = ( + train_and_evaluate_model( + text_only_model, + train_df=splits.train, + validation_df=splits.validation, + test_df=splits.test, + target_recall=( + TARGET_PHISHING_RECALL + ), + latency_sample_count=( + LATENCY_SAMPLE_COUNT + ), + ) + ) + + structural_result = ( + train_and_evaluate_model( + structural_model, + train_df=splits.train, + validation_df=splits.validation, + test_df=splits.test, + target_recall=( + TARGET_PHISHING_RECALL + ), + latency_sample_count=( + LATENCY_SAMPLE_COUNT + ), + ) + ) + + results = [ + text_only_result, + structural_result, + ] + + save_model_evaluation_reports( + results, + output_directory=NB_REPORT_DIRECTORY, + ) + + # 공식 운영 baseline은 structural 모델 + return results + + +def print_results( + results: list[ModelEvaluationResult], +) -> None: + + """터미널에서 핵심 평가 결과를 출력""" + + print("\n[Naive Bayes Baseline]") + + for result in results: + metrics = result.test_metrics + + baseline_marker = ( + " [official baseline]" + if result.model_name + == "naive_bayes_structural" + else " [ablation]" + ) + + print( + f"- {result.model_name}" + f"{baseline_marker}\n" + f" threshold=" + f"{result.selected_threshold:.6f} | " + f"precision={metrics.precision:.4f} | " + f"recall={metrics.recall:.4f} | " + f"f1={metrics.f1:.4f} | " + f"f2={metrics.f2:.4f} | " + f"FN={metrics.false_negative} | " + f"avg_ms=" + f"{result.latency.average_ms:.3f}" + ) + + +def main() -> None: + results = evaluate_naive_bayes_baselines() + print_results(results) + + print( + f"\n[Report] {NB_REPORT_DIRECTORY}" + ) + print( + f"[Artifact] unchanged: {MODEL_PATH}" + ) + print( + f"[Vectorizer] unchanged: {VECTORIZER_PATH}" + ) + + +if __name__ == "__main__": + main() diff --git a/data_science/SMSModel/splits/sms_split_v1.csv b/data_science/SMSModel/splits/sms_split_v1.csv new file mode 100644 index 0000000..4418b6d --- /dev/null +++ b/data_science/SMSModel/splits/sms_split_v1.csv @@ -0,0 +1,818 @@ +text_fingerprint,template_group_id,split,label,type +006cd7018b69018d6fcde02725148d6ccb30c7a6623a5f6d7cc07da24cff3750,tpl_006cd7018b69,test,normal,정상알림톡 +0091f7fd6db03e35ab34749c0647cf6cd5602bee395a64f471cf022e9d3708ae,tpl_0091f7fd6db0,test,normal,정상알림톡 +252bb04c80fefc85c1bce8d1af5e13679dacc684d44834d40f34432c6823adb4,tpl_0091f7fd6db0,test,normal,정상알림톡 +381ec9f45d6d6ea1c3e54b5d9a809c994dc7bdaab2dd5f8eba7df74365a3427f,tpl_0091f7fd6db0,test,normal,정상알림톡 +3fa53e1a0dbcef21354ccdff634cd39de46c7f971d77fb654f0464d282a379a0,tpl_0091f7fd6db0,test,normal,정상알림톡 +45d50423f1d4fcab11ad1ab2d6943b131136bdacfd388ada171db5411fb91ccc,tpl_0091f7fd6db0,test,normal,정상알림톡 +49d8c7da012f74d0e60991c50944874ac6f68821ad6f3d800ea2b5990fadcc36,tpl_0091f7fd6db0,test,normal,정상알림톡 +75b13b1e4921af2a1884f966372ab5073780131a0ccdab2e643b13535e139b24,tpl_0091f7fd6db0,test,normal,정상알림톡 +8de3ef9a8c3ddd13f244b6cf0f07e75c30321a06a98bd23421a9dc96a0aa686c,tpl_0091f7fd6db0,test,normal,정상알림톡 +8e5406beb5e67a1fbd143256333adf6068b724f05c19435942bca58885d49593,tpl_0091f7fd6db0,test,normal,정상알림톡 +db236d7525a63be5d076ccc3a14199a889d528877795d7c828e6cdfbca33bb28,tpl_0091f7fd6db0,test,normal,정상알림톡 +01fe49492e887f979035a5d103e9e2f95b7e971f5eb072ae0e5393e3fdcea16f,tpl_01fe49492e88,test,normal,정상알림톡 +024c7e6cfe9a6b242978eb66f744d63a01cec065f8463535baf9b7057ef23f90,tpl_024c7e6cfe9a,test,normal,일상대화 +07422ffe21a8e869596865b488d81592415a1ca70eb6e230ca5d65231cd1b9a4,tpl_07422ffe21a8,test,phishing,금융기관사칭 +181ae1ea5345872e0f28581d3963f0434961980da5eece94e1eb1c2de6f1ea57,tpl_07422ffe21a8,test,phishing,금융기관사칭 +7cf13c705ef8abee183fbcf7181e7274b495fa6bcfead218605cdef188242064,tpl_07422ffe21a8,test,phishing,금융기관사칭 +bcd778a5325414fcc05e69a21a8db0713405e045fa96b71f8f4cceac66b98154,tpl_07422ffe21a8,test,phishing,금융기관사칭 +c8a5374a9897f8c358c98668ad943a2b6af7460cf2a66af9c9e3e61ffb931bb2,tpl_07422ffe21a8,test,phishing,금융기관사칭 +d998145e942162310389538fcaf3a50aee2de27af11245488433a89de43230d2,tpl_07422ffe21a8,test,phishing,금융기관사칭 +dedbbd55a62e49d4ecb4826f7e6a2eb4a9f849d24b3e3a186ceafaab414eac11,tpl_07422ffe21a8,test,phishing,금융기관사칭 +f4483f9ae4ca9fa074bc0a22bb1df81612e22c0f838a11ee2d5a611e2b321c8f,tpl_07422ffe21a8,test,phishing,금융기관사칭 +f825164bc7dceff1ff164328b68c420c1ba21ea9eefdf90088032078fbca00dd,tpl_07422ffe21a8,test,phishing,금융기관사칭 +08e64badab3ce8d1fb4252fb85780bfe2a365447823d45cd0f78b78df89b9569,tpl_08e64badab3c,test,normal,일상대화 +0a7c3a3d728a06759004c8182e47512176080aa331e07bac4adfec8472e3dfa0,tpl_0a7c3a3d728a,test,phishing,택배사칭 +77624f5a437155c3e0f4ec1b2b48cf82c6974cf6e52f3f492e1e8f936e7a3257,tpl_0a7c3a3d728a,test,phishing,택배사칭 +d6b7fbeeb6c70ce0d26b41ccd51bf8c4f7671913230a0a4e0b6e168ce08fcc4e,tpl_0a7c3a3d728a,test,phishing,택배사칭 +0ba318438504c0a6d1b19344ca1ecab3a632255fb94b09c0c72dc29a6a89e4df,tpl_0ba318438504,test,normal,일상대화 +103fd0c676bf9cb692ca0babfed89f28288f96f944611c6562da9dd5a998e145,tpl_103fd0c676bf,test,normal,정상알림톡 +28551c15f0e692d859cb6eff3e498451bf936891d3b4c7c9b89a15ac9e26c4b0,tpl_103fd0c676bf,test,normal,정상알림톡 +392e0e1d00f5df3785bc159e271122835d7e665fd9eab7398752d7ea25c07673,tpl_103fd0c676bf,test,normal,정상알림톡 +413f35f8a63979441d39a25621c97077a9ae8d708cd851a2aa1e21ea311273fe,tpl_103fd0c676bf,test,normal,정상알림톡 +b59d8dacb0a9f79ca2cee06c85d9d857d5e7397659d04b9cca575faba6ae7af8,tpl_103fd0c676bf,test,normal,정상알림톡 +fc2ed5d52873a821c77b11d469604628d52db92da6268c26e46d181bb1c730c7,tpl_103fd0c676bf,test,normal,정상알림톡 +fe308bfd7fbaa4a8bce8339f6e246a72f7223160d5a9f5c662bb2e2f20fe74fe,tpl_103fd0c676bf,test,normal,정상알림톡 +136c599e8297d1d6683d54d540bd60a00116a915680f4d0b06251dcc421e1eae,tpl_136c599e8297,test,normal,일상대화 +14084b8ddfb7377e940b580b01b54edb58c92d0c8e7d1e19d5f671affde41a79,tpl_14084b8ddfb7,test,normal,일상대화 +1b4632392cc594cbd7f73cedfdb7cfe61384a76a8937ec34c9709142d490f8e9,tpl_1b4632392cc5,test,phishing,기타피싱 +1e1bca0b67e5836680f7d937566931992f6a174fe651f5dbbed1624a78a727ce,tpl_1e1bca0b67e5,test,normal,일상대화 +1fe36ae24d263143655e6255f205d25185df8214a2bd74e728e1f6bbe8f0e31d,tpl_1fe36ae24d26,test,normal,일상대화 +25f2a1bf431cec88c479a438086e0bed90fbc590ca315d70309f0311df20020f,tpl_25f2a1bf431c,test,normal,일상대화 +282f33a32e47f3cb73939db2c0c12b1a2546dca427f6f9b8645422d744382d93,tpl_282f33a32e47,test,phishing,이벤트당첨사칭 +29fbbd1a61dcc45e238443102a669e291dbb81a2a857f24ebdc3646686c49fed,tpl_29fbbd1a61dc,test,normal,정상알림톡 +2bdb1a1691c91d77464a2a27a3f38ef2a46f0008cac6b80834089a643bd790dd,tpl_2bdb1a1691c9,test,phishing,정부공공기관사칭 +2fbc6b4e11113b0d39ced22dcfda822d3b64760260a479766403efa1ef49e591,tpl_2fbc6b4e1111,test,phishing,지인사칭 +f40c0c0e16dc2519d56ebbed72df66b4d5230573ef34456dae9606440780c189,tpl_2fbc6b4e1111,test,phishing,지인사칭 +360922da735ea60da5223beb530999c60a74ba4a79e0f8ae010165f977bf02e2,tpl_360922da735e,test,phishing,경조사사칭 +e3bc2852b94e9fb1ce56dd6d7f2d55062837f8d421e281dead515b7e61fbac86,tpl_360922da735e,test,phishing,경조사사칭 +370820bbd3ac31f7e762e24ac225ba722ee86f88e776212f026452089688875e,tpl_370820bbd3ac,test,phishing,정부공공기관사칭 +3bf4492c0eec2c5bd25fda7462a3a3fa990882d1b0eab5920c595379e6b34c18,tpl_3bf4492c0eec,test,phishing,택배사칭 +5756f668f66b8efd83dd3c6bda2ea0ecc492319c2b876214c64412f3c2c4b8fc,tpl_3bf4492c0eec,test,phishing,택배사칭 +cae74472e95f1b714da33658ed4fc882f1a24dea1813c1430c8b7cb66da529ea,tpl_3bf4492c0eec,test,phishing,택배사칭 +46335873d6264088f204441052209785487588dec4113a6a4bcbfb7e62fe15ff,tpl_46335873d626,test,phishing,금융기관사칭 +47d42be0ab654b7a1241c60c8f37cbb3036e522db2731facc6751094986628d5,tpl_47d42be0ab65,test,phishing,경조사사칭 +482ef4cac8ac52e85cd3af5ca82f8b756d274a9f77631a630ae310a63b5d6a16,tpl_482ef4cac8ac,test,phishing,이벤트당첨사칭 +85ff48cd85229e66bbfab8aa5a892988e2f29dfe7725af9f0b34406b708b4e6e,tpl_482ef4cac8ac,test,phishing,이벤트당첨사칭 +49f1c13ecc2aec85aee3781f33abb1941fc4ce687292831d7f8da5c4e8c7a689,tpl_49f1c13ecc2a,test,phishing,경조사사칭 +4a805d171ae5a2408af652cca1360c3fb96f7105e02ab48bcf8eb4eb7455a62f,tpl_4a805d171ae5,test,normal,일상대화 +52364c242b35b4bfc9f984eefce92762db64e365f0da82a5a860ab50b398c8bb,tpl_52364c242b35,test,phishing,이벤트당첨사칭 +6699291fe2a2d6d7dfe29af2f6dcedc3df84d53dff895b836999266369af4c6e,tpl_52364c242b35,test,phishing,이벤트당첨사칭 +72bd25456633da234612e687c9d5e40823682ca73d74705f492f414ce47ebad5,tpl_52364c242b35,test,phishing,이벤트당첨사칭 +74e66b1de90ea895310cae70139278991682193f7b0e6849c79c2af5a33855aa,tpl_52364c242b35,test,phishing,이벤트당첨사칭 +9fd4afbb383e7071e6441fd7546498355d7996868aa7b1e5efe83dc18320824c,tpl_52364c242b35,test,phishing,이벤트당첨사칭 +a0618e370be4cd35a0948e13fc7b21d113a865c57697ac6a8cea625c792f570e,tpl_52364c242b35,test,phishing,이벤트당첨사칭 +f98f3e954a44240d4f764303e7e69f8ba8cc8f95cdbc61ef8ab0b410940ea3df,tpl_52364c242b35,test,phishing,이벤트당첨사칭 +53cdb27ed3c9645d33feade691a8d0a0fd2011975b2a7d03cfbbb28ce42b58a1,tpl_53cdb27ed3c9,test,phishing,이벤트당첨사칭 +9853949bedfb69ee1264769430cc307d1964e1df58fa40c33493ad2b58327b05,tpl_53cdb27ed3c9,test,phishing,이벤트당첨사칭 +549d0c87ec693085e33c5bfb17ce113e1220b093f84b6d675ebcc833a5901932,tpl_549d0c87ec69,test,phishing,금융기관사칭 +bc7b681f430be9f6abfcd8b2e82cd5acd1d377deb00a57ac1a639326337f8db9,tpl_549d0c87ec69,test,phishing,금융기관사칭 +56204dddd04e2f5320a3b40ddb0fa8c536feda41030c86bfb0c22443bb535396,tpl_56204dddd04e,test,phishing,정부공공기관사칭 +5a1166e862bbc39f76dc8ebbc623ed8a1ada4593904f861e0ecc116f66d02f40,tpl_5a1166e862bb,test,normal,정상알림톡 +5c5479d81f028e8c080726f771347387722f0274578c15d79ac39b123e5f6fee,tpl_5c5479d81f02,test,normal,일상대화 +5f3a11655d8186c91d23c63b75028732eae5a0a84079196553c3050b3f5d713f,tpl_5f3a11655d81,test,normal,일상대화 +601f80b3c1712d0c742ff3bea8b8fa670f5035d1670fb2e327f37d625e80847b,tpl_601f80b3c171,test,phishing,경조사사칭 +608e8d7acd6888da88437f8f51bbf97e138d94d6710138d1c1507e1ec08f1cf7,tpl_608e8d7acd68,test,phishing,기타피싱 +7361bcc4601a23c036cf990689fd4e8891a59e301d38cb3ccd41a8e92f5879a6,tpl_608e8d7acd68,test,phishing,기타피싱 +61f747f36cae729bfd1f65acd41d069ff31c580281aa7166390f570554809fc0,tpl_61f747f36cae,test,phishing,금융기관사칭 +6e612a61882b25b07870c807c306a2e7386884c9365a77e5b9920e0e6d9b3535,tpl_6e612a61882b,test,normal,일상대화 +742a7d7f9ed7c25982d3ed18249bae522ced2c7d06655df0adeaeff9b4812a98,tpl_742a7d7f9ed7,test,normal,정상알림톡 +7432beba9ae63a665af496879565356c69f499d54a0bea929e4ade88b5f6c357,tpl_7432beba9ae6,test,phishing,경조사사칭 +7811e8068f90f834422688ef168fcd72669d9a938fa26e6e90e13ea82386cb22,tpl_7811e8068f90,test,phishing,기타피싱 +79b09641fc8c18ef9d43e8956c47c96cebcea18d63a355b147e3d43822f74a09,tpl_79b09641fc8c,test,phishing,정부공공기관사칭 +7c42fc5fa6aaefbee4bf7cdbddc4ec6beb53e3ce7f1017edf86a28db3a80a868,tpl_7c42fc5fa6aa,test,phishing,경조사사칭 +7e803a109f408087cdb6cfd6bf68485ec85ef31a3c21a0e46408fdc145c60722,tpl_7e803a109f40,test,normal,일상대화 +82d58d02cbcd579703de5525cab7b7a10420a5b46ebf9a4e9b86351aa85ae1bf,tpl_82d58d02cbcd,test,phishing,금융기관사칭 +a75b37bab19f09933ca28a13fe7246b7e3f80055cc8bc4cc0edcd9f05bc30891,tpl_82d58d02cbcd,test,phishing,금융기관사칭 +844e591a312b51cf6c6c695ade131b9d0726bceff4fa805b7ac780a8807c01a3,tpl_844e591a312b,test,normal,정상알림톡 +8a49c70411785cf68db1adf84409ee9233b631cc3457c6812c4109c8e39f9307,tpl_8a49c7041178,test,phishing,택배사칭 +e0f25370733c7e2fadc7b1b56f10bf57bb631190b4be573aa628ef19f03b3eaf,tpl_8a49c7041178,test,phishing,택배사칭 +8aa6d45b86f5bf02a0c637c600d9266d85d76fc26d6c6811a1f601dd83692b3a,tpl_8aa6d45b86f5,test,phishing,이벤트당첨사칭 +8b12371de40c666c8e08355b5236b44d4c93c223a295baac937aaba2ef6ca0cd,tpl_8b12371de40c,test,normal,정상알림톡 +8df8048ab94380d6c2984b161623d1c980cec901df1cbf7b628fe3c6861d840e,tpl_8df8048ab943,test,normal,정상알림톡 +9264fbf807518d64347a1a67035e471ffc3edb2eb2215c2d17b39ee5628f2e60,tpl_9264fbf80751,test,normal,일상대화 +94666abff5c37cbceb44c9f3d78c9096a4c6f91c7e4c43c581a2a8e787760344,tpl_94666abff5c3,test,phishing,기타피싱 +95266279b6cb07b026952a8ef523d1897a141e4a36851ec4610b93afec3f3726,tpl_95266279b6cb,test,normal,일상대화 +9bdc1fc1cddea6433ca07b8281d257d5a2f0067dbd2d39f09a39bafbda2bd4be,tpl_9bdc1fc1cdde,test,normal,정상알림톡 +9fee15c3ca4caeae00867a6712fca417bcf361961c8075ca1aaa8f0fe15fa805,tpl_9fee15c3ca4c,test,normal,정상알림톡 +a4e0ae3d2d6c2869ed1693c3d6889cea119d3793c2c32d1ab6f84cff9eb497bc,tpl_a4e0ae3d2d6c,test,phishing,기타피싱 +a6c36ace707ecbb79e1d8081bf262c9ec72179931e3f4845e6f0988393880e57,tpl_a6c36ace707e,test,phishing,정부공공기관사칭 +a7b5c25fa4ec87fcf9375e62c26dbcd4fcee19e68953b5d3585baa61fad98086,tpl_a7b5c25fa4ec,test,phishing,기타피싱 +ab4dae7f98b1c79bf34586c3a1200cc268fa53f8fb970f5f18fbe461d5eb1302,tpl_ab4dae7f98b1,test,normal,일상대화 +ad9530e6816de3415f22b2ee6f6198f628a2d1a85123fa6e52c57401d623a8e7,tpl_ad9530e6816d,test,normal,정상알림톡 +ae20fd200d741e995f93141cfd6c1a9e67b5e0e9f930ce4de2825d7bccb57874,tpl_ae20fd200d74,test,normal,일상대화 +b09f8c35f1327bddc05a1c5286fe6c3581ff6eda995f511bb92204cd83d5dffa,tpl_b09f8c35f132,test,phishing,택배사칭 +b64fd98cc25a5b4dc35c12ec6bd28e26f545f04a3af2f20b603f842974ab18ac,tpl_b64fd98cc25a,test,phishing,기타피싱 +b7363c186f0cb511c69888940f9cd0aa664648d5bde5fdb393d152feece00d3c,tpl_b7363c186f0c,test,phishing,이벤트당첨사칭 +bac58b099fd068e975a6b06369cf4f17d8b9032f1aa8bc00ef8bf86b4e651a77,tpl_bac58b099fd0,test,phishing,기타피싱 +be188d78000b74332576edab22e644efaabbcc53f6fdfa682db844fd2f2c868e,tpl_be188d78000b,test,normal,일상대화 +bf3d685fcff1d14792e6bb05ab2ab3dffa7f6009bdf82e52976cda8a3ca6a367,tpl_bf3d685fcff1,test,phishing,경조사사칭 +c43b8807d618669045d21ea37afba443fbb9787ee4f215b02dbcfad29aa85f3e,tpl_c43b8807d618,test,phishing,금융기관사칭 +f30d63141b4845a656b058c240a2eb4d8f99c8f34da27131d70dce948618c2f5,tpl_c43b8807d618,test,phishing,금융기관사칭 +c54dfa3b24a493b38a0758d1e2081e0113661d38f18c2a361e86c4cc908e708d,tpl_c54dfa3b24a4,test,normal,정상알림톡 +ce0991338beca5a86e4ff70d5e773f1b6df05c05c82f0cd863c6df852fb66a1b,tpl_ce0991338bec,test,phishing,택배사칭 +ce40e5bed2be31fba6fb7e7513151e28f23bb6e0ca951b9844d4e5247c25c083,tpl_ce40e5bed2be,test,normal,일상대화 +d23abc29c1f752df4f56df1c0d76b58a7420f4894a3d8e2a074570d17b207245,tpl_d23abc29c1f7,test,phishing,이벤트당첨사칭 +d50d52b1aad042375f71df48993c969828676cdeaaf64ca4618b8f8e7baec471,tpl_d50d52b1aad0,test,normal,일상대화 +d6d43dd93959653d1fc82f4d2010f2e2da57f1c6270e2924d207132fc31bc362,tpl_d6d43dd93959,test,normal,일상대화 +dd9c13a2c2f2747da2a12764030c3b1df5dbeb94dde8116dd4b4cdf9e93df172,tpl_dd9c13a2c2f2,test,normal,정상알림톡 +e4b4ab5311af85b227ffd8e935ec6449d97dd3d95a253e7431bc325a7323429d,tpl_e4b4ab5311af,test,normal,일상대화 +e80b5d63b72e124ed19008d477838c9e6832d7ea7f4e6c85b8f3f2eb8e912fd6,tpl_e80b5d63b72e,test,normal,일상대화 +e881629a268028296798652bf8502a9fcf032f1888bd3f7f4c8707634cb2a999,tpl_e881629a2680,test,normal,일상대화 +ea80497f3c18c315a409a6d208b9cd91eed75ef16462c1ecd601414b8666a07b,tpl_ea80497f3c18,test,normal,일상대화 +fa7e40410d49530f9394e39be2a26050582d04bdf265ce2108ab92c369c3bec0,tpl_fa7e40410d49,test,normal,일상대화 +fe62346937700a270b99cefc32883cdff7e232df1ca400df44fb9dd435b504d6,tpl_fe6234693770,test,phishing,이벤트당첨사칭 +00036b47f69b5ab13f95889dbd7396d4dc6c6c466914a27123b395b984ef32ba,tpl_00036b47f69b,train,phishing,이벤트당첨사칭 +0a7e68806e59bd63634e4ced5c9a5956d1f64b9feb90c02ef6c9dcff0178f40b,tpl_00036b47f69b,train,phishing,이벤트당첨사칭 +385a22210a95c36a0134e0507c3b84f7d1df9faf265b93617842f6b769b382cd,tpl_00036b47f69b,train,phishing,이벤트당첨사칭 +409717c789fc9776d56fc9760ce9291463695e4367cf05fb680e9d6319e997e0,tpl_00036b47f69b,train,phishing,이벤트당첨사칭 +4887afe8ee562454d8bc7c8ace61c3468b6977164488ae9a110c5bfe74ac56b7,tpl_00036b47f69b,train,phishing,이벤트당첨사칭 +bdcf4c13ee46528638bc8951bd8466946be8c0bddb92197112a4f353e4d9b1d6,tpl_00036b47f69b,train,phishing,이벤트당첨사칭 +009cc31f75fabda2b4bf23fc81c1b537f93f4c5416a590ea03c4b884c4470d88,tpl_009cc31f75fa,train,phishing,이벤트당첨사칭 +00cf157f99e0615f9a5a173db43a74e75564e208859861bea5919284aeedb9cd,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +4c46c7dc68d9a3b80fec68d5cbb554f770833d24a6fff0b8032204d0e1a8e055,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +5236dc1acc90bca0f285696566fdac0f05e1cda164ef1b7a6f470426aa1b8bd9,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +c5275a2a4229ad43150fe4d7d353ce766705cfeda906e3823cf348f11800bb3b,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +d53a987f0563fa60dfda1b5c12373a7d4a8287ffe97084c011c4e94d2f9a4b04,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +d668f85bcc3d93774d2dba524831e43dc3d157da10a20e0b1ca60960ab0dd3cf,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +dd8c26d7a566b97cea6bf05ca04bb11d3e57dda33de0f9cc8d5e1438420f2d33,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +e3d680c5299c6f539e5bef927b72e5dc54deacee2a852fe8e53c0d21bfa051f1,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +eb027607260431acaf8405018e2637627d199fc72a23ca12c60042787266e893,tpl_00cf157f99e0,train,phishing,이벤트당첨사칭 +01934901ec9cc96dd57a4b38f7cdaab152df13f0c40e074511a50f11925eab0c,tpl_01934901ec9c,train,phishing,이벤트당첨사칭 +f647aed9a429109b6d9f1170db1e26abe7efca0dbe62d51848d064a4e36138e5,tpl_01934901ec9c,train,phishing,이벤트당첨사칭 +02f481b00a2b0543ba0c632c3ef6d611863ad3c936ad0be31b893262cbfb1df1,tpl_02f481b00a2b,train,phishing,금융기관사칭 +3131ff4abe4d4eb37aea49b9686bb93f8cd7ada4a53ff497b0880c23f67a2d9b,tpl_02f481b00a2b,train,phishing,금융기관사칭 +77bf186ff581e9c4eb8f70513befcaa408625d52275b9b5c84893f951c4d5d02,tpl_02f481b00a2b,train,phishing,금융기관사칭 +8498b9939f687ba50559ff341f3764b8cb405486fb0c5af65e1c88d46fbac8b2,tpl_02f481b00a2b,train,phishing,금융기관사칭 +a29d05d605cce62c051e5632916c0a7cb6b14e11c3c99955324cdd1756d03bce,tpl_02f481b00a2b,train,phishing,금융기관사칭 +c626003a48c87684f5cb5336c52e4fbc4dd0e8108ae9240e1bca4240e2ac51c1,tpl_02f481b00a2b,train,phishing,금융기관사칭 +0388439e40301f717fd84073613bbc0fa99722a7f4b482ae97180d4f8e79eb03,tpl_0388439e4030,train,normal,일상대화 +03d59bf1111617477a3cd283de6c9340b5358b90a6e6b95cebe01a106157b724,tpl_03d59bf11116,train,normal,정상알림톡 +055222934ac2de1e414bddcb7cd7f59d3cd3921141a039e0448abb867f19a9fb,tpl_055222934ac2,train,normal,정상알림톡 +3fcf48e4d208808933ffa8ff1d2622f041394b125d571ec44160bf77e5d9d7c2,tpl_055222934ac2,train,normal,정상알림톡 +9113934be52f8c542b48370d79dc387059b068a23efc9bbfb9fc6a3dcb3fc226,tpl_055222934ac2,train,normal,정상알림톡 +df91d4ca56f159f59b81b60135c819fd7206a396eff1eb9e8197acc396a99e9e,tpl_055222934ac2,train,normal,정상알림톡 +057441485d917a9ed636777cb6bc83abfc93fd756cb1bd82c62e0ec8f8f96248,tpl_057441485d91,train,normal,정상알림톡 +0925058e8c0f92db41c106a0e37226ca9478651769c1bc7bb31defc2c7693dab,tpl_057441485d91,train,normal,정상알림톡 +34e9647604756e48542f65cd4e63844868ce650e3d65e3e121e4970346285b92,tpl_057441485d91,train,normal,정상알림톡 +3ef0a3b62064d137ba433ce89e28bb80a7e2110d05b4afd1ec42d955ee4464da,tpl_057441485d91,train,normal,정상알림톡 +5ba7e2dcab1e944127011db13a9f9415ef0319a3e3d84e532ffbbf732c638851,tpl_057441485d91,train,normal,정상알림톡 +76aed99e0ef513c7c1dd04f989a703782e81c2f2b7142f96191c7f5cb11c4581,tpl_057441485d91,train,normal,정상알림톡 +76ef0bd88bd6431a7e5281abf4b65c2d2c8f9c76eff72908e2179d75a6efcb70,tpl_057441485d91,train,normal,정상알림톡 +82c56159caf1b7c26ec4c5949b589742b2f47a9d34bd5756e1b077449afaa080,tpl_057441485d91,train,normal,정상알림톡 +c0bec63a392cdeb6247c166211094a20e2df6e572ae7f1c9625d7e8b9e9393dd,tpl_057441485d91,train,normal,정상알림톡 +d7b6557dc791566d62c6a72dba81d2c22ea0179cb1938fdb80358fa7e2c0815e,tpl_057441485d91,train,normal,정상알림톡 +e5afcd5dd0a20eaa2bc5490b63504131b147d6ffb66a4917778d0d4fc02c061c,tpl_057441485d91,train,normal,정상알림톡 +e82e8ce9c29b1558623fcbedd049ed5c1b686f4848d77cdaac8ce05c5d646cc6,tpl_057441485d91,train,normal,정상알림톡 +ea3417e3339eae8a97370cf435fc029b281529e12709aa696d5d1d72389eef1e,tpl_057441485d91,train,normal,정상알림톡 +f07519f53a104e09a5a728c638e10e8a73a5d9175cf03fa6330a5c60da24fb34,tpl_057441485d91,train,normal,정상알림톡 +f79cdba42c62c2631002a8f13d2fbf7dd25681ed3c36094199fa271440504b0a,tpl_057441485d91,train,normal,정상알림톡 +ff920cdec80a3a63000ae14cac754f54808a0dc20887c0fb6ddd41245d8983ee,tpl_057441485d91,train,normal,정상알림톡 +061121ea6e5a60f9512b6ff0fee3292a4adf8ed5bf1e7c3b9edd4aec854eee0c,tpl_061121ea6e5a,train,normal,정상알림톡 +1c63e295924146345469a5bdb7bea4e2c651e4be050af1f4ee241fc2b71436c2,tpl_061121ea6e5a,train,normal,정상알림톡 +0664ce67156ce04829df8ff3413da46c3ecdf48674594506efd5cf4ea692b4f5,tpl_0664ce67156c,train,phishing,금융기관사칭 +37c94216714af5fda0010ba16f10b1dd5502f693bac4ef84b35a93a535321ef9,tpl_0664ce67156c,train,phishing,금융기관사칭 +3977b850df7d569cec4092075e91ea786800ff0835afbb55f9287fb2d2cf2f70,tpl_0664ce67156c,train,phishing,금융기관사칭 +52a0a2312e8f0647186a76def39b78b184e31c21bce310a61f154aab8378989e,tpl_0664ce67156c,train,phishing,금융기관사칭 +5e8c0a9554568c3069eb306f4cccfe93de4a5ce102a4f8f27c359b6e3e2e4082,tpl_0664ce67156c,train,phishing,금융기관사칭 +8b3840c41715d056dd4ad9ac45e9e8b38e454e0b0a3c4f56ccaa09e510fdda69,tpl_0664ce67156c,train,phishing,금융기관사칭 +8bebb1a55a876217b8c3a02dadb50166cb7fde906fa09ff35099a01b46f0e059,tpl_0664ce67156c,train,phishing,금융기관사칭 +d5c1f60e37f3b985c494e304e4421d6134a34f5b189745919d77a8949ddff57f,tpl_0664ce67156c,train,phishing,금융기관사칭 +f1f11027a57811a6bf3c99730a4477378f6ff3abea9f7331aeca3ff56d61f013,tpl_0664ce67156c,train,phishing,금융기관사칭 +089f434eadbd9ee638a2af407ac901970f9bdfa58a820274d5401890cc49a774,tpl_089f434eadbd,train,phishing,지인사칭 +14369042657028d75aa8b4487e36c40468452f5de2ef89d3653ce2fea290a0a0,tpl_089f434eadbd,train,phishing,지인사칭 +27e5343dc1a043dd397e96d7f3575d7e984e4eb30cc1f45a52ca4d0d75b67315,tpl_089f434eadbd,train,phishing,지인사칭 +3b29cd15123e59726f6a6872acdeabb364bc689b1db71a8e4a7518a123e54dcb,tpl_089f434eadbd,train,phishing,지인사칭 +5016d36327eb0b14213d89bcf52726b864309aed3bc922e670369e1f1c9e4f85,tpl_089f434eadbd,train,phishing,지인사칭 +5a7695e0da4086abf2be3bf501f0ede5d4e8956537cb1f4e4499b5cb1b38d686,tpl_089f434eadbd,train,phishing,지인사칭 +5ad7f397c50fa0f4a0ae6b167de25501a4f80f60db19d1b8245aaf3b46fde715,tpl_089f434eadbd,train,phishing,지인사칭 +68f363ca4c772910ddf1cede0cd42625dc6006df406907f2c406df7a964e8cc3,tpl_089f434eadbd,train,phishing,지인사칭 +6f10fbf2407f409097eca60353ee7ec2a3181f1fb92f248ee541832fd1c3d47e,tpl_089f434eadbd,train,phishing,지인사칭 +7ff615276d8085048d16828ea56c9bb909c081f13f57ad4f9fc225db887275fb,tpl_089f434eadbd,train,phishing,지인사칭 +81f6312061468d48ab4cdd2fe7b7c914d90e378a307de4febab41812b4938f14,tpl_089f434eadbd,train,phishing,지인사칭 +9152d7e5544da9ec4569be5cae54c3c7f8635bf1a5b56042458622b03fb6a827,tpl_089f434eadbd,train,phishing,지인사칭 +a040db216ab4e3e6d4f8cb92ada9bea8c5a8e69bffedc6447e160ba50a3f4efc,tpl_089f434eadbd,train,phishing,지인사칭 +a306fc5d9ef664032dd3ae39359e597aff531c31c136c4cb5a4f3341bd52c520,tpl_089f434eadbd,train,phishing,지인사칭 +b1516847797fc4cd5b55cfecda544e5a2cadf11d3dce40a2dbdea3f0937c8742,tpl_089f434eadbd,train,phishing,지인사칭 +cc01bc619accf8bee1868df96f58d882f3f8f2be6356020f4920698240192b71,tpl_089f434eadbd,train,phishing,지인사칭 +df396b209b38864c755a64d4c99d52addb800d7d51e42fd4f961b397cc01cc4a,tpl_089f434eadbd,train,phishing,지인사칭 +e0b2806f12401614ffd37832c5f9980538cc955caae62eb0acd4312bc07f68d9,tpl_089f434eadbd,train,phishing,지인사칭 +fed7cfc5b7b281601ebae5005dac8695468c0d43e0b2c220cb8f7c1afac2992f,tpl_089f434eadbd,train,phishing,지인사칭 +08c236ace2da65c7acdad4c5cde47b89ccc78dab9e73511d02c841598ec0e12b,tpl_08c236ace2da,train,phishing,경조사사칭 +0922eadc5a7c34cbc9fe3f045df029d27098101b0ef92d46f59e19fa7d58af10,tpl_0922eadc5a7c,train,phishing,이벤트당첨사칭 +44cb4d03bbb9000be403e09d24d5605ba6d43d54ca6f2c572137fb019cf1edd2,tpl_0922eadc5a7c,train,phishing,이벤트당첨사칭 +092ebe92b646037397c774d3040b9a6114947b9c18ff8e3a10e7e02f58d41871,tpl_092ebe92b646,train,normal,정상알림톡 +09b57f1f6800f96c668cc997116795221de247730d5a432132cbce2089f6621b,tpl_09b57f1f6800,train,normal,일상대화 +0a3f686bd31714f7740c7f91e4761024e351d600bb4fb125251ded429f08ddd2,tpl_0a3f686bd317,train,normal,정상알림톡 +18d20d2efa8df4580a1cc07cc785ed24ce1ca69d60f38575927c95940e05006f,tpl_0a3f686bd317,train,normal,정상알림톡 +375356d6803c13f4be91908f8220b7f56b370ead5a04b446d0ef0e1c3945fc3f,tpl_0a3f686bd317,train,normal,정상알림톡 +5c592f4b179588f01e19e4d2f9abf254bd4367fd16279f0d30d85082ada24294,tpl_0a3f686bd317,train,normal,정상알림톡 +68b556611de433b70c7d30e8a151ed623193f44774a6a3493e14ec8333c0cd91,tpl_0a3f686bd317,train,normal,정상알림톡 +9e7f78fa2d245016e03d4b9c2736cced573cd253da46f1a57aee500b9218b419,tpl_0a3f686bd317,train,normal,정상알림톡 +a5d44e0c58b8a7405592371d3269b95bcccbaafb9bd7a010294c4561f14867c8,tpl_0a3f686bd317,train,normal,정상알림톡 +e72457ee803b744d5a6495adf968ad93ff7276649b8271ac5e6d2fcd8fff2641,tpl_0a3f686bd317,train,normal,정상알림톡 +0a97cb36007c0aa95a9c92280832d3944e63702c74e75f4394b90ff0f4ee0774,tpl_0a97cb36007c,train,normal,정상알림톡 +0ac682d241a4d1354d7236185262f4046582be584c495781d549c50b706fdd6f,tpl_0ac682d241a4,train,phishing,기타피싱 +0b20e50ab9efe7cb2dc98b0b601c246f651913449812b10ee9c4b40a19e8ec1f,tpl_0b20e50ab9ef,train,phishing,경조사사칭 +0e615393364bc7f64b3820c0adda8cbb335058ee79028d415183d861e3f84036,tpl_0e615393364b,train,normal,정상알림톡 +202ae0604e045296aaf9607396c0d0930b0324236c7b179e35138e5705ca452f,tpl_0e615393364b,train,normal,정상알림톡 +2f2502108c4181956a65110cc8f5237a78fb8e9f5278201a2f898320e196f565,tpl_0e615393364b,train,normal,정상알림톡 +374a22505f11d0bfd26dd4f7eddaeab5cfd1978dad6aeacdd99760345d9e7889,tpl_0e615393364b,train,normal,정상알림톡 +805ed16a91a4291aa0621c3864aea2818c628200d2a7bbd5d5b8c6a2d5254c7c,tpl_0e615393364b,train,normal,정상알림톡 +996b10e4d1bdf451c52d001bc98f09750c3dd0765dc17d89e9380ead94a64fb2,tpl_0e615393364b,train,normal,정상알림톡 +9f314b447d1d696796d23e14d27d9313385b8cc8e96dd0ff24e8a24d65351bce,tpl_0e615393364b,train,normal,정상알림톡 +a2a54c20baa1a847779c115c12177f9cbf83747640d965818f69e09c31195f11,tpl_0e615393364b,train,normal,정상알림톡 +a647053db5c41bb55c6e15a84768bffc19517d315d36f5e738d25c67cb58d819,tpl_0e615393364b,train,normal,정상알림톡 +b33f4f2137a05857ba776ee7fd6cb21bfcee0113a70b5c1c539baab0578df19d,tpl_0e615393364b,train,normal,정상알림톡 +f5500e8b3f44592d1162b6fba49096384fda142d0bf0ca48af7ff4a621ad08e5,tpl_0e615393364b,train,normal,정상알림톡 +f5d4c2c1554542ff5672baa711b88287c5f0f79ca58864c339451caa2034884a,tpl_0e615393364b,train,normal,정상알림톡 +0e9e7583ef9c24a7f30b5740f09bf65750d3b64a94ee90be1e82c1b578155b39,tpl_0e9e7583ef9c,train,phishing,지인사칭 +12a7b4dfa09d40904522c050d50b7a4c806037439787f34129d6a33d22df8bd8,tpl_0e9e7583ef9c,train,phishing,지인사칭 +20a6f24359d5c28727dc54693b3572bed8f5806353b8aa93ca7b4c38ca545db2,tpl_0e9e7583ef9c,train,phishing,지인사칭 +216b9cec872184b64341293d4d0a0367de196b0f20d2948b124768c1a687c214,tpl_0e9e7583ef9c,train,phishing,지인사칭 +220ea2045a8022a336f39be9fb72f2a5d173bb209b619462d2b7e6bcb4582486,tpl_0e9e7583ef9c,train,phishing,지인사칭 +3dc6bf3496277a248c351cb87b679ed8518a70ec42a9746392977c6e94d7aab5,tpl_0e9e7583ef9c,train,phishing,지인사칭 +549bbf1566a34518b2e76aa91204d81beb3be268dbece7b57718a52466f72f6b,tpl_0e9e7583ef9c,train,phishing,지인사칭 +5dfaf6519f0381574f8f7bd2443d6337ca6c0ea65cbd365c2ca909fdd46d95dd,tpl_0e9e7583ef9c,train,phishing,지인사칭 +68b603a4f666c9bf0f0fb7ce75e61133f811982a8e1fe7cf93fdd194f1db4cd4,tpl_0e9e7583ef9c,train,phishing,지인사칭 +70c6b2a92d0dae8efa3728a62babdfed8672ad8a59025bfa5e8c56c0a59ef4fd,tpl_0e9e7583ef9c,train,phishing,지인사칭 +b27e184dd0488f7a036af64664e4c1c6685627bccf81c3a24e85c0163fb4169d,tpl_0e9e7583ef9c,train,phishing,지인사칭 +b53dffa51126b6aaa48d0b47dc95a0d74b4c0b71f4ad9c2166a05ff141c72d75,tpl_0e9e7583ef9c,train,phishing,지인사칭 +cb44b8b2b544e9c0065e1ce60be0a613255ee3f2f6146dba860bbca8387f39f8,tpl_0e9e7583ef9c,train,phishing,지인사칭 +e499d81479605de27d20a73bf3fd860a893d946b51aa2f6bf7bf6d5e70b2c5ed,tpl_0e9e7583ef9c,train,phishing,지인사칭 +e7f2a9608fc5c3f3c4820dabc93b10155f4a3f0a96da5b689d33b9a3f5d36a3b,tpl_0e9e7583ef9c,train,phishing,지인사칭 +e815384ce15009efa520db3ab1ea463471f03daa50aa49ad77de212bfbfb833b,tpl_0e9e7583ef9c,train,phishing,지인사칭 +eb4c48d85f5bcd8f8a6bb348ec1f96e05d01c09209134af79f93a6d500076a2d,tpl_0e9e7583ef9c,train,phishing,지인사칭 +0f7a1eb398222ccfd9bc97bc6f65a8698d3dad8e067bcbcec2cbe667394cd9ae,tpl_0f7a1eb39822,train,normal,정상알림톡 +2c482ab2ff393f88d5b2144df4c22344e97b92307a293e865d2368fb048575d8,tpl_0f7a1eb39822,train,normal,정상알림톡 +77ca9e371f65048cea692b3ee13c0d86f0813a89951aa1faeca57aaa5ff9be40,tpl_0f7a1eb39822,train,normal,정상알림톡 +9b77c718e33ee197cc17024cccd4a50f47f21cfe15ce5b567d17ab062f5f4c9c,tpl_0f7a1eb39822,train,normal,정상알림톡 +b1f75330ab8fc75fd667b6ac51b52a96b0d4a1770d326a81997bca8166c5acd5,tpl_0f7a1eb39822,train,normal,정상알림톡 +ce0465a929cbea46e845c4dcd6cef01124fa422e25089745b3053b8d7341569b,tpl_0f7a1eb39822,train,normal,정상알림톡 +e748e32b4d5549f62a54e6840f26c39c27a0552921482b776ad05dcd482135fb,tpl_0f7a1eb39822,train,normal,정상알림톡 +ff563b40d4ce9beb926ed827dcacd8f6db01706a49dc62df68084af873272a90,tpl_0f7a1eb39822,train,normal,정상알림톡 +0f7b4f711f926479fb007fdedf122ee4e0867d9a3282dee480db9d405aedaa24,tpl_0f7b4f711f92,train,phishing,경조사사칭 +2158617405629be233934e542526dcc423ce9431a742546b8ba09921a93232d9,tpl_0f7b4f711f92,train,phishing,경조사사칭 +0fd4ccc6966fa1a56cf451c487fa7493769a0c03c500bf34cd56d5a1be367d11,tpl_0fd4ccc6966f,train,normal,일상대화 +10230255369903082a88920fac2895b0714dec5e6ff7b2a729be4d23ace84d51,tpl_102302553699,train,normal,정상알림톡 +102ecec1e3c66b3d3b2fee43a9b766dfbeea6807a2862be01d6e115b97ab909e,tpl_102ecec1e3c6,train,phishing,기타피싱 +115d89dea69dd5d72cb7ace6381a1503856d9a7ab1eface3b07e42afe5234840,tpl_115d89dea69d,train,phishing,경조사사칭 +12814d2830c096378317300e2421fddbe004328e851f7c64680b9f336dab4ce6,tpl_12814d2830c0,train,normal,일상대화 +12f9185e83be994f4a8198f89a33770e588f3eee5ff258f8bfe1bcb6f80233c6,tpl_12f9185e83be,train,normal,정상알림톡 +13a9d1849e20647058ff4c6f7da202520e3cd230aad38a24629e20471e5590c9,tpl_13a9d1849e20,train,normal,정상알림톡 +14375491e8f89328be6abc7f20d6abc8c7184ba6198692b3dbcee39715f951b6,tpl_14375491e8f8,train,normal,정상알림톡 +145b37457a53ef2273e72ef6b9361c86622bc9e8b4c0842dd2a4b870bc586c4c,tpl_145b37457a53,train,phishing,택배사칭 +1a36e1dbce2227ddadea0a333c3eed4bd8c8fd15ae691cdd07566e96581cf361,tpl_145b37457a53,train,phishing,택배사칭 +5971223bfc5bd8165ec5238e0ff2ff555b917035d6d79b03d274dcd225c64e1b,tpl_145b37457a53,train,phishing,택배사칭 +7b36832d322f5ead7975a2b1789b8f8d93a08e6244f9ba28f35083e3b1f71b0d,tpl_145b37457a53,train,phishing,택배사칭 +1647e15d83cbbfaea7b9d22eef6246ac5d7e8ffa78e6ff9d28ce6388e69cf7f1,tpl_1647e15d83cb,train,phishing,기타피싱 +2f16f2e8c0022d80292b990beeeaa7048e2bd1f4987a054bcf1dcbf1f952ac12,tpl_1647e15d83cb,train,phishing,기타피싱 +567b61eaf54806da49ba7085194d7ecd1e99d55cc9cf4ec002f6bc2c0865cf52,tpl_1647e15d83cb,train,phishing,기타피싱 +6e2b29a829a129053bf8f50528faff5d6c510dae94e9f6e71a513682ba39d81b,tpl_1647e15d83cb,train,phishing,기타피싱 +730c13eee53bf80a677e9182f16bb4a3ecae6c01313f7beceeb29032830c2606,tpl_1647e15d83cb,train,phishing,기타피싱 +7b74ec6aef2bcb33b5e0835f14e5ed7c4e502d48fd78b12fa68f29fb30138afd,tpl_1647e15d83cb,train,phishing,기타피싱 +855ba18f219ac65fb2a580a0f4b26b6d273fccbe916969b1b84143c54c8f23f4,tpl_1647e15d83cb,train,phishing,기타피싱 +8844111ca682afcf5caf8645b0dfe2baf16b6b0a95bf64cf89a0f0aebb55ee43,tpl_1647e15d83cb,train,phishing,기타피싱 +8bc4a6980a8eb1ba497b53431bd548580b08520ad6d8d488e515feedcb6b1425,tpl_1647e15d83cb,train,phishing,기타피싱 +a675e160cf2437041289204afbed8250f986db40489ddbee641657a4e82ca470,tpl_1647e15d83cb,train,phishing,기타피싱 +d6bd6292e14c2fb5769a40381273f1c92ca3dbf055e6c91c62605a7148be315b,tpl_1647e15d83cb,train,phishing,기타피싱 +dc3cfbe1755d7803359907d8e69ae4b1760910330b0166a71859d3658cefc250,tpl_1647e15d83cb,train,phishing,기타피싱 +e5b85d3e28c979753116396f1464bb3db9c90da3e743f77b39f778bb70ed69ed,tpl_1647e15d83cb,train,phishing,기타피싱 +f8bd6ec14f3b32d7cbab9fdb32d713e11c999159c907aac195c7539da539adbf,tpl_1647e15d83cb,train,phishing,기타피싱 +164d2883399b2a8767b728418210208e6167a9deb5d7a33ee31022f4480e0bfd,tpl_164d2883399b,train,normal,정상알림톡 +16c413bece09254458058b4098fc1831ea9d916bf384cacd362cf1d7b61e3c9b,tpl_16c413bece09,train,normal,일상대화 +1717bc55259a1f8214b2c3b3af4fd234e59bd24bf774b35d3bfdcfb62c70d96f,tpl_1717bc55259a,train,normal,정상알림톡 +17a65e70d02f95c3ab410159c7ebade91394414a9e1b7626f16afd11a347395f,tpl_17a65e70d02f,train,phishing,경조사사칭 +191048c04de2bb0777adf6d705c12b7a6b025cec9ecf8b513cc1dfea872201d7,tpl_191048c04de2,train,normal,일상대화 +198e76272478e31968162b7eecf5fb43cdb0867e8b56209ca3bd9ee8e90a78e7,tpl_198e76272478,train,normal,일상대화 +19a937ca4936ac1384f70666640ea89db38a00cda523629f316e73566c86e881,tpl_19a937ca4936,train,normal,일상대화 +19bafcfc5053e1a6e36d15b1acf1022c5dd6ec73705cbb5bdeb0f3f0772efc83,tpl_19bafcfc5053,train,phishing,이벤트당첨사칭 +1af351323caa019f9ed7b5319fbd37271d28b47b309e6ab268dfe6bbcf03f3c3,tpl_1af351323caa,train,phishing,금융기관사칭 +1b538383bd482eac3211546f3eb77fe3340f71e1a507f1727bbcd7c5554e4ba7,tpl_1b538383bd48,train,normal,정상알림톡 +3ffe7351bbaae187b623b7d3f36883ce33291d3bd21776a744e1ce49899884be,tpl_1b538383bd48,train,normal,정상알림톡 +4500e37aa1d42963021b06531c284b87f20d0c6f558084f04a52a9a34fabb42c,tpl_1b538383bd48,train,normal,정상알림톡 +7e3aabb364ce57866b881da6ef7dd6b953268e05267a323c72a50799df39731f,tpl_1b538383bd48,train,normal,정상알림톡 +95adda6bb56eadb664fafd78110e4ad88f22880471679ffb558176b6da1c3153,tpl_1b538383bd48,train,normal,정상알림톡 +b7d667bc51f5c9fe97cb257c1e513e5f019c61be26f6771dfe8843883a5bf29d,tpl_1b538383bd48,train,normal,정상알림톡 +e142cdd243127df2be02681d70ab63c0727d0f5ea21b8fe01e32aac6f0a5f7d6,tpl_1b538383bd48,train,normal,정상알림톡 +f9cf49989bd91792719aa506fa701db83db29c35863fcb7e555ca854d14ba637,tpl_1b538383bd48,train,normal,정상알림톡 +1c0aaa52af0969b88856a21bb42a468113cc4a0f27d8adcc49cf321320ac0a16,tpl_1c0aaa52af09,train,normal,일상대화 +1c83d987803dc35ffa5be9f64b5bb516aa9568f255af29d5eb00ace32d179db7,tpl_1c83d987803d,train,phishing,경조사사칭 +1cbc1c63cb36f924cd240ce001b6574c88f31ce7b7c59202d234e86415172478,tpl_1cbc1c63cb36,train,normal,정상알림톡 +2a53e3e8dedd808213585e7e84c840b0d6b2b401d8739e47cf950f9a897039cb,tpl_1cbc1c63cb36,train,normal,정상알림톡 +43baacffbac2d00edd0e2b7db8f5b0545a249b5c29f699cd4599faaf2fa0d33e,tpl_1cbc1c63cb36,train,normal,정상알림톡 +54c91fbd2d2509afca8c29bd6eda30302b8aa0b6a197f9e2e748c3eb876da03c,tpl_1cbc1c63cb36,train,normal,정상알림톡 +5dc0c00e30379a033731800c8ce74ced3311ab3fa8a0972cc03c1aa097240254,tpl_1cbc1c63cb36,train,normal,정상알림톡 +7a6c336558ae738c7ff7843312ba0868911489c0a906fa9a4ee6f40429af4078,tpl_1cbc1c63cb36,train,normal,정상알림톡 +7da6a9cc311e1fc29d507fa983f300754af55fe3c0a77afce5759f21ff2782e4,tpl_1cbc1c63cb36,train,normal,정상알림톡 +8591b3b79410484e4375ba301300d05e00f08ff37241158e5cada8e5eae7eab3,tpl_1cbc1c63cb36,train,normal,정상알림톡 +ef92fb22275d0be45e01466f2c2ef2dd2ea06f133d6c3862a9ab75e45c63fde3,tpl_1cbc1c63cb36,train,normal,정상알림톡 +1ce25a24d7a52abfb22090805ef33850f2e3962ee7c0c1723d324973f718b421,tpl_1ce25a24d7a5,train,phishing,이벤트당첨사칭 +1cff6e040fb461bf49ebeefd8a881c81063858886e90d7aca0232e70437fbc75,tpl_1cff6e040fb4,train,normal,일상대화 +1d5710a4969d96d30a2d6e6c861d5e0c969f11cb072ef6e3948ed8bba87f8b89,tpl_1d5710a4969d,train,normal,정상알림톡 +1f3ffd855eee608f10adb86ddb88a8d261876161d61c055f17fc4e813e4f69b8,tpl_1f3ffd855eee,train,phishing,경조사사칭 +1f4fe57afe9da9182c20e6c8c424e7be1d7bb04d75b9cbc7b59293afe8e2fb81,tpl_1f4fe57afe9d,train,normal,일상대화 +1fe6fb72e51fb430979a4928e8165fc3327413ebaf4db9be822a85ffc21a7664,tpl_1fe6fb72e51f,train,phishing,경조사사칭 +1fed23922041561e7e1f8a07f25a8c653e02da9f17d99b2a00fb9adac9e4c55b,tpl_1fed23922041,train,normal,정상알림톡 +20d3b679ae2b03fecad489dcc16361839eb15ce60a2863e45845c1a6a1909992,tpl_20d3b679ae2b,train,phishing,금융기관사칭 +a420565bb2c475ca64c3a9ab3ca2fdaddc7d7a387bd39f4cb63aa4867f14bc39,tpl_20d3b679ae2b,train,phishing,금융기관사칭 +21514a9a939e642ee85898e00fad7599a002dfce65817f5671aed8e61542cf67,tpl_21514a9a939e,train,normal,일상대화 +2156329f036113c89e3ec1e1060eb3c09a2303a0a4a2f058d73ed7f634d634c2,tpl_2156329f0361,train,normal,정상알림톡 +220fe590396dd47b434ed2e493338488a89787befe46d6c2fb30a42803f6b323,tpl_220fe590396d,train,phishing,택배사칭 +7a1be0de47ed419dffca4501ae634b74cd966c225bdccac3dab90761f2f08759,tpl_220fe590396d,train,phishing,택배사칭 +faa6b3c04eb3e66f6c1399b5a7b08611ec7e965e3d4b6c1612c9a2c5088158fb,tpl_220fe590396d,train,phishing,택배사칭 +22233f743d0a45f1fa694381da9eba58798d2b77088077649e9f57ebe8493786,tpl_22233f743d0a,train,normal,일상대화 +227f9fc663ef1991db2027e7dedcfe74bbbcc0eb352eb47f38d84a1b0838dba3,tpl_227f9fc663ef,train,phishing,이벤트당첨사칭 +ee3cde61975d002ed286ccba5d59c82403a39a896a04276c07a5a3d73f16b7cd,tpl_227f9fc663ef,train,phishing,이벤트당첨사칭 +23576f72f508b95b4cecd9e89f7a18b040e04dbc0456086c27a79e89f5d0f1ee,tpl_23576f72f508,train,phishing,택배사칭 +847d1495221bc4c9773db490537d70563700e5bbbb14026708494b17da918b19,tpl_23576f72f508,train,phishing,택배사칭 +2469459fec79ae93ebdf059172065bbc5870b82586cc685b281ee516e4bff09e,tpl_2469459fec79,train,phishing,정부공공기관사칭 +24838bb1566216fe6b9a78f217362500e1bcabfe51dae3651d68f1a205b5016f,tpl_24838bb15662,train,normal,일상대화 +2717c495b40216b748632e4da0eb0a30dc073b9c8a77582aaab21ac0b7ea1539,tpl_2717c495b402,train,phishing,기타피싱 +28c1dc759401d4dfbb6dadb0c9a4af81eeb267968561edf467c3e0f8a0f4800b,tpl_28c1dc759401,train,phishing,금융기관사칭 +7bd7be2459028c882ce097f2888699bb6acb891f2b5d901163ae68799cbc8c10,tpl_28c1dc759401,train,phishing,금융기관사칭 +2a84843ef0d1bddc907f39596cd45537159aa56de86d906888dfae0373ae88be,tpl_2a84843ef0d1,train,normal,일상대화 +2b2bd4c45551756af858833575827ffed2b275a9c098651947354d6a54c7f79e,tpl_2b2bd4c45551,train,phishing,이벤트당첨사칭 +399a9dd23bc2721c744e66b47a3dda68ea014d389759d321e789dcf85ec25db1,tpl_2b2bd4c45551,train,phishing,이벤트당첨사칭 +69b477846af50c596beb778658f7a63c6ab83e7769dcc8683d6f29f1b00719d0,tpl_2b2bd4c45551,train,phishing,이벤트당첨사칭 +86366f74f763defafaf47e5011e76653439731f01156a4d1d8aa4dda2ee1a770,tpl_2b2bd4c45551,train,phishing,이벤트당첨사칭 +9c3d323f805da246c4d9de85734e096f33b087528d91df649c1cfe8db15c377a,tpl_2b2bd4c45551,train,phishing,이벤트당첨사칭 +a3472ca0bdc6f99eab9979f0588cc55e2568d141fac4b5fe7508b0b3c7914ca3,tpl_2b2bd4c45551,train,phishing,이벤트당첨사칭 +b6917519cba2a40834b95ea6fa3928965a9d516789498b2581e4e5ea2da3faf0,tpl_2b2bd4c45551,train,phishing,이벤트당첨사칭 +b86963dca71859059c9c38ccb4df0c1409c6836f87be40fd64d361f5e790bdeb,tpl_2b2bd4c45551,train,phishing,이벤트당첨사칭 +2b87d37edbb31660f18191e28c3d781b22edc8aab835077a9d39972829a815b8,tpl_2b87d37edbb3,train,phishing,경조사사칭 +2dd0ed87375938a736264bbf4ef7c465c0b38f416a896dc0a597b5bd2bd2d22a,tpl_2dd0ed873759,train,normal,일상대화 +2fffbf7d57c9b81a5b88904f3a0187a9ca5c5c67a2fd9ad66e4f2114ed58b92b,tpl_2fffbf7d57c9,train,phishing,금융기관사칭 +9102892db4ea1fb98a56bdb0ba8b3798d7e00dd16114e3117bfc098907e1fa0f,tpl_2fffbf7d57c9,train,phishing,금융기관사칭 +3009c14cc0320ed7467577761135d3285828136e21745fdadce2775c5fb41a6a,tpl_3009c14cc032,train,phishing,경조사사칭 +303d0d960990ea4f6a8b7f198574511895d080de5ca3fea42d04db387185109a,tpl_303d0d960990,train,phishing,이벤트당첨사칭 +4cb63f056605307ccd0c5165ac7c87a4a18a487abd5d5b0e003ca5b861f30486,tpl_303d0d960990,train,phishing,이벤트당첨사칭 +666ed1deef49b6ebf5e4e00596d707b18410d5263fe93f9d15b5fd537228d6cd,tpl_303d0d960990,train,phishing,이벤트당첨사칭 +702bb0b9116eef9c716b3e13db8f39cecc767c3e6357a2f2600385e5c607ce87,tpl_303d0d960990,train,phishing,이벤트당첨사칭 +fc976cb4832b678d26ddca1cd6e3a472fe0f36e25dc88b9cd15e926df274e89f,tpl_303d0d960990,train,phishing,이벤트당첨사칭 +30c231c0994a0094b2acdcc2a198535ad4df24ab4b61967a17e03065c901eca6,tpl_30c231c0994a,train,phishing,경조사사칭 +30e8097bdc1813eae34b105c20b4c92722a0d09cc0f212e1ac91fd32bb5e3831,tpl_30e8097bdc18,train,phishing,기타피싱 +31f528c2be28b050ec1569e00463eb09deb4f26fa841a18860b8f2fde9091cb9,tpl_31f528c2be28,train,normal,정상알림톡 +323eb6cfe5370f66a50ca0fd5a7a8424bd729c9057d163470162bff008065973,tpl_323eb6cfe537,train,normal,일상대화 +328d84c0ed29268e75d47fb6f98080502dee8bc2eda08cf92f2c9324d649d67b,tpl_328d84c0ed29,train,normal,일상대화 +32ebfadcc9ae33421da8f8246b2309ec4be9846cd569d9ed45df0407b714acc0,tpl_32ebfadcc9ae,train,phishing,기타피싱 +5d9bc43f9fc6069106615b11d80ef6471b46d313e089c334212a2b6b306337f9,tpl_32ebfadcc9ae,train,phishing,기타피싱 +32efb6e9341c3b23d5be8d7e33054a9e137bc48dd43742c1f0d191f5d53fb4c1,tpl_32efb6e9341c,train,phishing,택배사칭 +330d6bc0ec647b5fba27876d80cdef79765306f85437ab452ffd6dad32a46bdc,tpl_330d6bc0ec64,train,normal,일상대화 +3311c1d93d93cb5f10443da906257614c6ead8ee3de6573f26547509ae5209cd,tpl_3311c1d93d93,train,normal,일상대화 +3364a7396e2c62b05bce5f42c49fc9697859159b21dbe384d9932e592de43e9f,tpl_3364a7396e2c,train,normal,정상알림톡 +336f2ea0bca911f36db33934af9478bdfda1660eef31b80671f00b237d0026f3,tpl_336f2ea0bca9,train,normal,일상대화 +340643ac085f401dad843c15572b1f5e9d150c89a13a0c90a4002bd8437eb127,tpl_340643ac085f,train,normal,정상알림톡 +351fb86fcc96ee08de77043dd172eddf79f3db3218c3445de6bd9f424aa07044,tpl_351fb86fcc96,train,phishing,금융기관사칭 +37066b8b214b607a0acb56926543b03f09e1b11afc0c52709d9b61f5bb4ebbfc,tpl_37066b8b214b,train,normal,일상대화 +374de891b052dc90bdb456111ba752f7b407129009ec0c7188a2ad4cfa0d4fd5,tpl_374de891b052,train,phishing,금융기관사칭 +383ae02c9a3f76df11b9af58e3b9290c8dc9fe36a8b156e5d39e84293e13fbe3,tpl_383ae02c9a3f,train,normal,일상대화 +38688459a512928b680657cd12033f6a780cef17fd6153ab6220b18c6f69e4b8,tpl_38688459a512,train,phishing,이벤트당첨사칭 +38cefc95912a9eedc16b6882a36769bac4046b90ee24b3bf68f6251db7c6edf0,tpl_38cefc95912a,train,normal,정상알림톡 +39d05b840098461e72dbe3cb63b91e57183d6f65df1763509605164611f872a8,tpl_39d05b840098,train,phishing,기타피싱 +e8fd0ea787a039d859c2ab5c6282d183d9fd820b4337c6d45a7a444e3df8c4d8,tpl_39d05b840098,train,phishing,기타피싱 +3a673e8aa52e3229530c772391cc97134a90588359b065e5dfc1eb9e7ee862ab,tpl_3a673e8aa52e,train,phishing,경조사사칭 +3a6d89a14192f975036c05dfb3805f9dd5d60bb204e49af34bea05102d990f76,tpl_3a6d89a14192,train,normal,일상대화 +3a9d8fb8ae15bff657376c41a7b5f167435dc4a237dda54e503d894d3a0c9fc5,tpl_3a9d8fb8ae15,train,normal,정상알림톡 +462f9e2588e6766079f659638629c75a62d25ab79554cc6efef23687d0810398,tpl_3a9d8fb8ae15,train,normal,정상알림톡 +5ad5b9a5a0ccac50101757bc42636299f5d01ebe8399764cf1132e5bf5f2e9d8,tpl_3a9d8fb8ae15,train,normal,정상알림톡 +6450ca7e146bba2c492a0811da17a9367b503493c5b03f704f5dc78d1c52ae40,tpl_3a9d8fb8ae15,train,normal,정상알림톡 +3b777d3dcbe4123c9beab7a4f1e6f039244e7b2014f9b1032e3a972270c131f1,tpl_3b777d3dcbe4,train,normal,일상대화 +3c25f5477084e2c54eb6676b971663a70418a8f228670032ed46722988a97bfc,tpl_3c25f5477084,train,normal,일상대화 +3c4f03e033db5485f2f667e208ea7e83dbb9da061dfe1e83710b0a2234d8cd4c,tpl_3c4f03e033db,train,normal,정상알림톡 +3ee0a37ade1893c371b3555108b9388af0646f85d86ef79b187d2585efe5d447,tpl_3ee0a37ade18,train,normal,정상알림톡 +3eed09cfd9610e37903a044c8cf7495dbf70af911e25ddccf9e79c00de72692d,tpl_3eed09cfd961,train,phishing,금융기관사칭 +4035ef6b16fe4b9ac5b570b31dc7949ba937b96288a4ab3aea86334649a57232,tpl_4035ef6b16fe,train,phishing,경조사사칭 +40d9fae89d485e95e4f166059682c864c1b01a6aae887f848e4b418c00a32f4a,tpl_40d9fae89d48,train,phishing,택배사칭 +51172090023c0f4b95da019e8f246e0291388b836ec724542bb5095b76323857,tpl_40d9fae89d48,train,phishing,택배사칭 +40effa08f76d5f67a25a0c0369c47e88bc4358eafe60b72c8314f986459ecb8d,tpl_40effa08f76d,train,normal,일상대화 +4104aaa81bceae1d9c6b3a92e8a3397ccae5282d1e7d3bd2d74410918ddf9050,tpl_4104aaa81bce,train,phishing,금융기관사칭 +bd4ad638c7a946c6b6ec9dcaaa0d0c1e6ad486f4e2ab76dae677e5814bfcaa7c,tpl_4104aaa81bce,train,phishing,금융기관사칭 +4179f54dfe26365994664664a8621684d1b47818b3588d81bd890449acf218ee,tpl_4179f54dfe26,train,normal,정상알림톡 +419613331a478d59d0ac4cb9f5595442cdded1dd8d9e2954bd8bdb2b87b0e936,tpl_419613331a47,train,phishing,이벤트당첨사칭 +eb3a7989d2e65d507c0c7ef906b75e0f72921eb3cb2fa7c4f9daae03eec2396d,tpl_419613331a47,train,phishing,이벤트당첨사칭 +41f5bb6108ef76446fbdb8e19be660b9ff53d89b9e9d1b5e90847deae2c1a24e,tpl_41f5bb6108ef,train,phishing,금융기관사칭 +426bf67c21cfb764413d9498ec94efe6f94c07c4999a4c5bd9591e63f5b263c4,tpl_426bf67c21cf,train,normal,정상알림톡 +5807b7d3d2784730e3fa2498d6a5602c9d394b091d0a0aa1df3c5e9323906236,tpl_426bf67c21cf,train,normal,정상알림톡 +78690649e236da13080f383a4fd0f81b4fbeabb9e9a2684e751679c854473a20,tpl_426bf67c21cf,train,normal,정상알림톡 +8c30ccaf935e05203e168e08ed304704a908a7f91e5a6217d1596e6ccaa91ed1,tpl_426bf67c21cf,train,normal,정상알림톡 +fded9f3fbe3d5f29002813c835ac6a0e6a88033e9785cf5ac7424f4df3eb0711,tpl_426bf67c21cf,train,normal,정상알림톡 +4302465d305b43832c7d61cc779ff5881e7da7f65a73f0b391b70b286a9ef455,tpl_4302465d305b,train,phishing,정부공공기관사칭 +43a63c163c267c08858106925ed8663c3714dbe125c8f3f9b5af73dc8ec87a14,tpl_43a63c163c26,train,phishing,이벤트당첨사칭 +a4ec8cf581e4102ada80d58afaaa61c601d2ce960510ba98eeaf0213c20cd72a,tpl_43a63c163c26,train,phishing,이벤트당첨사칭 +44fd09fbcb7634026a775a7b9c9afc9dbf629868d9948995e19d00c6954967ba,tpl_44fd09fbcb76,train,phishing,이벤트당첨사칭 +459f91d709a6bb14924f640205afe1652a45c18a6b3a5aff6881e7f6f62805a9,tpl_459f91d709a6,train,normal,일상대화 +4634d6b801d170d4b3ebe096dfc563826d60fdcdde0936929f997ddf70b6cfbd,tpl_4634d6b801d1,train,phishing,경조사사칭 +998b63e2c0478a1033bdf5fb474f79620f7aff9f73386b82c7e5f18de31d9dda,tpl_4634d6b801d1,train,phishing,경조사사칭 +4738b05ee8c32e815c7d0f25d58e7895167b7befc837d9497547cdcb00122d8a,tpl_4738b05ee8c3,train,normal,정상알림톡 +48f4a00d2002afdae755933fcc3731e1794dae8f311d0c8e463c9061314e4172,tpl_48f4a00d2002,train,phishing,경조사사칭 +49225758d75143d92c6efaa5acf0b58588bf8ae9fb51d7804b4f299f4ea7f9f7,tpl_49225758d751,train,normal,정상알림톡 +497a4125d43a77d0a81bfb61833ed65e914ccc66ea754b08479a58cf570d41b1,tpl_497a4125d43a,train,normal,정상알림톡 +4af3dec3810700b693d758bc58550d2a514ebcc887cd4c3b30d1fec489c4bda2,tpl_4af3dec38107,train,normal,정상알림톡 +4c7fda436b34ccf88385386c0d404cb1c55cf48ec819328e6762eb1c7fa400d8,tpl_4c7fda436b34,train,normal,일상대화 +4d023258d88d38acbb4cdfcdc1cbe3ee102bb8f0bf25bec528ec524ad37200ce,tpl_4d023258d88d,train,phishing,정부공공기관사칭 +4e01b5facebd00057d71eb151793bfea9754455e2492ee8debebb5a578852cb6,tpl_4e01b5facebd,train,phishing,경조사사칭 +4e7bab1aec8cdc7b203a66eda88f48747507329d351b34bee743318f22a174ef,tpl_4e7bab1aec8c,train,phishing,경조사사칭 +4f3d089d5282f4d469d6da093ccf897cbcbd11389db14f6d860fffd22f67b26f,tpl_4f3d089d5282,train,normal,일상대화 +4f497e918721762642d62776843e758fe66b7c594b937b4c16fee9697f49e56b,tpl_4f497e918721,train,phishing,금융기관사칭 +4f7f017a61c7a6de76b6b9131cd4493970308b5a478450a66f84d621e08b6f08,tpl_4f7f017a61c7,train,phishing,이벤트당첨사칭 +65e21e09acf4753067e5322d87134ee2b6e227441dde88095011bb375b414954,tpl_4f7f017a61c7,train,phishing,이벤트당첨사칭 +6a293c5f730c2ab975419074cca20c0085b411189639a9a612efe44effe75a22,tpl_4f7f017a61c7,train,phishing,이벤트당첨사칭 +4ff495bbe562a1800db7979c72952abd7c05a9a814bd14b19e786e21633b6780,tpl_4ff495bbe562,train,phishing,이벤트당첨사칭 +50be054364561b45221d7753dcc85c7dc91c6cd7e3c308dcaa2eb2d755cd3eb3,tpl_50be05436456,train,normal,정상알림톡 +93e782c55684e9c040430f62f4d707b1f0c36e9f7ec8334b88baa7a023f9aa63,tpl_50be05436456,train,normal,정상알림톡 +d207f6d4b8786fabfd6a43b699c13ed08b547778c8519e83c3bb31f9a1e56512,tpl_50be05436456,train,normal,정상알림톡 +50eea112cff8345cee16a55b831a1f2b87c99a8d295e4d624f1151d50ae097ca,tpl_50eea112cff8,train,phishing,경조사사칭 +51ca4ada569ec3290d4d43a3e0df9494795604a0839b8f3c7c70a8d2e02837e6,tpl_51ca4ada569e,train,normal,일상대화 +528ee66f67f11a1f7f6f77561e42d5ce6e67c6fedafb867d431d069079859315,tpl_528ee66f67f1,train,phishing,택배사칭 +774b157940595be6008989290ec47cf95b4ffa04cd65de58147613e904d7904b,tpl_528ee66f67f1,train,phishing,택배사칭 +5357c9b1ab69168a7e4b658494a14f0146ba11cbba76172bb8c031404f5e8edd,tpl_5357c9b1ab69,train,phishing,기타피싱 +537c1de6a5e3721f7ead8729753bdd226f4cf4184329323eb77aa132170b3e22,tpl_537c1de6a5e3,train,normal,정상알림톡 +53b6793759bc9ac25312729b59ca058061ad682033c3c48bbf7d6263709efd84,tpl_53b6793759bc,train,phishing,이벤트당첨사칭 +547ac5870e2c7246cd96dff3aa538c7148343c6d0fed5344aa621024104cd106,tpl_547ac5870e2c,train,normal,일상대화 +548e485bfdcafd221a48c1200afd9ef10757b6f29dd00591473fc6da9f9aa16c,tpl_548e485bfdca,train,phishing,기타피싱 +56a89ae5613ef03858cc535f927bc916fa5ddd5b5552137add476676e7301d62,tpl_56a89ae5613e,train,normal,정상알림톡 +56ec01a8920ca12fa6afb869214d09f51433f13591b59c9f227b814df2047846,tpl_56ec01a8920c,train,phishing,기타피싱 +5719ad163e2e66c931c6c8543e5bf4a9a93539ccb9a6f491640c5f22b3ad2cd4,tpl_5719ad163e2e,train,normal,정상알림톡 +5788ba18aef47f24a0165795eae832eb7598854c5703e8497c5bc65dda02a50c,tpl_5788ba18aef4,train,normal,일상대화 +57cffe9714b1345b9f2cd6377ae3c0b4151905a733861cbbeb51134a078e46e0,tpl_57cffe9714b1,train,normal,일상대화 +58fcdaca8d268fd66d117b51d0f2709140e7f3a4ed72072ab330481fd0ac7f00,tpl_58fcdaca8d26,train,phishing,경조사사칭 +5928d47de7d591fa513b33f445095e7bdf6e4156257d054d7daa871fee874140,tpl_5928d47de7d5,train,phishing,기타피싱 +5a40c3f63bec6e17241934bb1eaa1f04034fab09c6fc824f10926c01fc71e43d,tpl_5a40c3f63bec,train,normal,일상대화 +5b8dd637c3ee0635767cc72e15b04b8546eb6e1aefb9fd4c2dce7daebc99f040,tpl_5b8dd637c3ee,train,phishing,이벤트당첨사칭 +7794c21098dea3353468b11e178ffe8c17c033100847ac66915963db0b69eba4,tpl_5b8dd637c3ee,train,phishing,이벤트당첨사칭 +aa1d8c1b4b8c554b77f8d62c7d82584b491fee403b0d23fa421a9763b4bc423e,tpl_5b8dd637c3ee,train,phishing,이벤트당첨사칭 +d1bc17e433c142d71ce7d485f6e3aac53d5ab143d5116a76379f99e6b27e2d86,tpl_5b8dd637c3ee,train,phishing,이벤트당첨사칭 +5cac45fd0afdc096318548b3a06032cd938689d86025d9e81e2e3fc76e14b33f,tpl_5cac45fd0afd,train,phishing,금융기관사칭 +5d1845b93bc6271e51f1f4da6ed7bbb7eadbceeaebb1bccebf1d99823c7fb717,tpl_5d1845b93bc6,train,phishing,경조사사칭 +6027c81ed0a6e4c34bff7c53517ee0082b51f818ecd4ba42383e812c2e536894,tpl_6027c81ed0a6,train,normal,정상알림톡 +6042b81b11cfa5a8c9db730e483d9ca119afda3caa0a9e2831fa3ca066689d92,tpl_6042b81b11cf,train,phishing,지인사칭 +614a880be44d0b2f10635f5fca645439da84840f4e16ab13fb833f87a9161b0b,tpl_614a880be44d,train,normal,정상알림톡 +61ccd047a6d5f78a71f945165a65d40f04f8c92424d699d6319c5af94682c7fb,tpl_61ccd047a6d5,train,phishing,기타피싱 +6264471631770bfab21e352b3126b91dee8787bca02870769ce9b97df995e40d,tpl_626447163177,train,phishing,경조사사칭 +7494b250bc4e12947daa3eeadb9e95d4bf008c9bb4299aa23331385fcbdf0416,tpl_626447163177,train,phishing,경조사사칭 +7df5fd97208f9d712b611f8686532c8774a2c19a38199b6a02fd83bef1603d56,tpl_626447163177,train,phishing,경조사사칭 +c32f2b849f62f8c10ac8d54f80544d3f4f9303eb67a9e268afcb7ea6e0121b3e,tpl_626447163177,train,phishing,경조사사칭 +633013467fa46e34e22678855d4fa9c00529ff5a963f17078286a6e4ae6e6d4f,tpl_633013467fa4,train,phishing,정부공공기관사칭 +63fac6a7e948aa2daef3f106f0315a48e2361f6cd07047ce725966ccd42075cc,tpl_63fac6a7e948,train,normal,정상알림톡 +64b5f6913730673f34a3350a0404737b16d3454990d4ea728d9a377b16655bb4,tpl_64b5f6913730,train,phishing,택배사칭 +64b7a8d4db71b7a64cb6f64072624c5a6ea92fad1fbaf0fc6049a355841ec62c,tpl_64b7a8d4db71,train,phishing,기타피싱 +658bd0302ca24225db5dcede38402e21f0f4261b6150e6845fb99217c88363f7,tpl_658bd0302ca2,train,phishing,기타피싱 +66476be9c9a8b6f3848bc8c50c7a6f39978149ce720ba276a65216c6034b7cdc,tpl_66476be9c9a8,train,phishing,금융기관사칭 +7b7faf097479cdd22c3797254ac77150c7ee3dc25281e0ddad2ff2ecb15d8c7a,tpl_66476be9c9a8,train,phishing,금융기관사칭 +678043b76f34b1e4a27f89296423e86d3b3fd0a69ccc60bd06e2d65dbe945772,tpl_678043b76f34,train,phishing,기타피싱 +67d4419ebf73e190f9269eba71b37e3c3fba296655405d8c0b0cb44facc2f37e,tpl_67d4419ebf73,train,normal,일상대화 +68aef00e02e7133e5ce1dab98dce0f515820a43377f4ba270aaedf63f09c0754,tpl_68aef00e02e7,train,normal,정상알림톡 +69e6335d5210172a8d51e7f519a353c266357c5f857679c75a82f108877d4f99,tpl_69e6335d5210,train,normal,정상알림톡 +69f97bb6c8973d864ca30434d19ec6e7fa9ff2900b017c07d47f4af10b43fff3,tpl_69f97bb6c897,train,phishing,기타피싱 +6ad99290e53acba1f66ae22a8ebe5fc9f585d25f892bf4aea5d7e88c372d656d,tpl_6ad99290e53a,train,normal,정상알림톡 +6b59c4e7c7200785a81cf1630a232d8173b9ca54015b28d1933462c23b6ff028,tpl_6b59c4e7c720,train,phishing,기타피싱 +6b818d1bcc889de492158cbb27cc6fead5196d4d92e194550c0617b9ea94c9ce,tpl_6b818d1bcc88,train,normal,정상알림톡 +6b8530a4a30f74cb07d8a9ab79175e7c7afc544d6d1592be218f4bcf620a6fa9,tpl_6b8530a4a30f,train,phishing,기타피싱 +6c08ac772a9035ede986d290b13013d642abc683ef31fb0218708889c21d787e,tpl_6c08ac772a90,train,phishing,이벤트당첨사칭 +e885858f910cc6aa1a41e1d3f02b12059cea5d7a78e6babde4f6d2efda4e5cbc,tpl_6c08ac772a90,train,phishing,이벤트당첨사칭 +6d8c86de04bd82ea5eaba651b5cdcf54225145ecdca25d59739c5b4cbdeed358,tpl_6d8c86de04bd,train,phishing,이벤트당첨사칭 +e2ae0bf52273bab58cb078da3f0edb39a183b5e72c69c2937cd09dbed97c7ec0,tpl_6d8c86de04bd,train,phishing,이벤트당첨사칭 +e34018030ffd539baa9f577b4f715f6e202d07f9cc141d3f3963a37433ae0ed6,tpl_6d8c86de04bd,train,phishing,이벤트당첨사칭 +6e3832f8300e8b81a6519c542e89e405da7e6711a258d9863dea0769fc571153,tpl_6e3832f8300e,train,phishing,기타피싱 +6eb0fcee38ba2c8eca1c35e6e0634837d398b55634645ab8bc76895f85e22983,tpl_6eb0fcee38ba,train,normal,정상알림톡 +6f15f28ab51707a01eece16b101f17e8de759d6fc25e335deb7308157fd0b636,tpl_6f15f28ab517,train,normal,일상대화 +6fa1e69a8254be7c75cd5e96b48c258fa15c39fb1195f102b78555f5cc7f9911,tpl_6fa1e69a8254,train,normal,일상대화 +73a84ac2a0698124c0e02f93c8a62ce381812580602852e03507561cbb8a461c,tpl_73a84ac2a069,train,phishing,이벤트당첨사칭 +73f7dc0cbe8e31fee604c0f4ddd7983bcd9b093c9fcd7390a9260e4a8b6c7008,tpl_73f7dc0cbe8e,train,normal,정상알림톡 +740f666aa55ba32be0a4497f55b68d7252b36af6a7f6b007a5dba150c343a141,tpl_740f666aa55b,train,normal,일상대화 +755cd5c67c401b1d968ccef0ce00fbee534bf9998e4c83a8db2b2004f236f3a8,tpl_755cd5c67c40,train,phishing,정부공공기관사칭 +758734630c21b4a086097893a1d8288cc3ba6d62dab1bbf3085820ef9e7cf197,tpl_758734630c21,train,phishing,경조사사칭 +77278a2e5683eafd5419c0275eb61bd2678328fca4fd4d5934aed15d50930fca,tpl_77278a2e5683,train,normal,일상대화 +7776a4e0889d7f31127be77d87188d7d4c74719d6f4e44f73490f46c81ef7e12,tpl_7776a4e0889d,train,phishing,기타피싱 +77b81d18dc5de0e4539eae4029aed9b4e8c1dd6ba45dd38469449f488a4ce4d2,tpl_77b81d18dc5d,train,phishing,기타피싱 +78262fdf9fdeb08e69c258e8effe297885de8fd6679704e0e24e6fd6b58dd173,tpl_78262fdf9fde,train,normal,일상대화 +79f9661919d5f78c3246a4ea5bb5bc33ba4b9594a619197c1799297552786111,tpl_79f9661919d5,train,normal,일상대화 +79fdeed36aa7c05d7d9b2d80c4cdb1326f812e59b472b6b7d71590e2ff58a371,tpl_79fdeed36aa7,train,normal,정상알림톡 +7a248834ad5198f1966bf1623614d0826191c285f23d662428a572bad02a8e31,tpl_7a248834ad51,train,phishing,정부공공기관사칭 +7a7244609a658e5720305b4a6c09bc488152fe7e2e23dcc280e945ed8de7cb6f,tpl_7a7244609a65,train,phishing,택배사칭 +7bed9ede02f4adf17b1136b17cd241319edd95a6cb28004e5aa03af2523d9945,tpl_7a7244609a65,train,phishing,택배사칭 +dfb58de0f8a8dcf3226f840d2e329bfb53946e8afdec1393f8cc6de9d8c33cb1,tpl_7a7244609a65,train,phishing,택배사칭 +7a8f337f72c98d7881a78e8cef3f18ef1d9b0aeeb34c7decfde1170dcb2f7a61,tpl_7a8f337f72c9,train,normal,일상대화 +7b0fa86ddd02acd80b5d66ffb2609681e984678c71ae0b3f0f6ce849388202d0,tpl_7b0fa86ddd02,train,normal,일상대화 +7cefdecd35f063b74a7afc9092963aa396de7970a395909ab57511d6a028f5a3,tpl_7cefdecd35f0,train,normal,정상알림톡 +7e11cf72bd5262b522fba4c090709cbeed0508014269b26f6ec33d0b479157e4,tpl_7e11cf72bd52,train,phishing,경조사사칭 +7efe8828c26b022fec3993a5a29f8a81b07fdc29f07b15e1c1cb871b2c2ca966,tpl_7efe8828c26b,train,normal,정상알림톡 +7f7df0856fc59c690f8b800e2a5e6711d0f2b9491166c77164ce44fe65735002,tpl_7f7df0856fc5,train,phishing,금융기관사칭 +7fc907ff40a862b9f7c6ea34bc358720b10c020e044006d4bbcbbe90332feb20,tpl_7fc907ff40a8,train,normal,일상대화 +81ede268b7b4dd0b1c5adbb70578b843ac3913830837509b0af87d8f114e9f2e,tpl_81ede268b7b4,train,normal,일상대화 +82434c439d6065e9c46a0af3c8e14f460e2526eb41dd0acd719e5da32c0e7b9c,tpl_82434c439d60,train,phishing,택배사칭 +829fea59062a71258522b9c5cc465d68b271418a21b667825e9e41e699928603,tpl_829fea59062a,train,phishing,지인사칭 +83c88eea9c16ab346ce9cc0dabc6322546f7f444347d98281b90873b21845792,tpl_83c88eea9c16,train,phishing,정부공공기관사칭 +84334b779d175af9b56b7585774a9ea557d49b73da1a08052e7e185d778bd856,tpl_84334b779d17,train,normal,정상알림톡 +84ae70f72910886e3b2a77b08b897d2c190470f05cb6732bcfd3b82b92dbf6dc,tpl_84ae70f72910,train,phishing,이벤트당첨사칭 +84b99849ff317d41b8a682af294c0b5081ba713718bfb627f267a82ff107d349,tpl_84b99849ff31,train,phishing,이벤트당첨사칭 +85d5ffc30c98a576e4ee63d081e91f22994a43389550110c683538af64e9c8ce,tpl_85d5ffc30c98,train,normal,일상대화 +868c8bcd2e5d71324167e64188eafad1c3c6d115ec08d33f87e6c6aa5e52d2fe,tpl_868c8bcd2e5d,train,normal,일상대화 +8811eef6e85c1c3756927a38f87c2aa221f9b9d88e25a1fb316cf9c9da707772,tpl_8811eef6e85c,train,phishing,금융기관사칭 +903448b5dee4bcb27ae78bc57458fe8dd35cacf3028e8026e1ea5d9977b48e22,tpl_8811eef6e85c,train,phishing,금융기관사칭 +88ef80d822eca8cfe738955e75cc0efc7a0e89ece5162a4319a71ee17edfd798,tpl_88ef80d822ec,train,phishing,금융기관사칭 +8936232b3de4df7cc89ec91c42203752f9643dc7f21c722551a8c8101794a405,tpl_8936232b3de4,train,phishing,이벤트당첨사칭 +89a3aa09779e79164ca091cc601492e2a04c3b3df48b4f946621c7840b05a458,tpl_89a3aa09779e,train,phishing,기타피싱 +8a10480d36601bb95f55e9060307b80164c2b2bff9e167cdbf229c78df9e1698,tpl_8a10480d3660,train,normal,정상알림톡 +8a57008dcdbcb0bf6d641d4db49f04916b2bbbdc4c1307cc7c290342d52746ed,tpl_8a57008dcdbc,train,phishing,정부공공기관사칭 +8b2c2657a3dbdce16eab281d981ef4075aa33e336c7722cc2f6824166edb2ad3,tpl_8b2c2657a3db,train,normal,일상대화 +8d442c40db321b44488e196c9079860eaa7d9f61bf014737581a0b19fa77a370,tpl_8d442c40db32,train,normal,정상알림톡 +8f61ade1c7792bd329f109fed7a11c4de284e53845e4e1b47bb1cfe43fdfbce0,tpl_8f61ade1c779,train,phishing,이벤트당첨사칭 +900b6706739a4a9c67027eb00c34c64b8a073b62bc95aa3f667dab2e19d45560,tpl_900b6706739a,train,normal,정상알림톡 +9051e19c4f680760f3b01ee7588381052957eafe6b5d8769105f54fb8309bf69,tpl_9051e19c4f68,train,normal,일상대화 +90f9886f06fc64ce53782b64e743c284265937f5ee39a3163b9496c27688491f,tpl_90f9886f06fc,train,normal,일상대화 +91e21c583f16abdc42ab8cf4a6c2cdd63c3b80e5f5a2d6d2b0ea765a8f3c2067,tpl_91e21c583f16,train,normal,일상대화 +91fda0dd98c106982336793ce09dda251f00e166fc5831707122cd5b6b6446eb,tpl_91fda0dd98c1,train,phishing,기타피싱 +92802eba144a092041874f6d3dc8e2c0631bbefba947a3a72cb7022a2f729a41,tpl_92802eba144a,train,normal,정상알림톡 +928cb457932441a4bccbf68334635089fa94fcad1c56f0f0264372960fc2d2ae,tpl_928cb4579324,train,normal,정상알림톡 +9397997bc5d97edb5de9a4c887e8d0a260eac8df3f9b2bdbdac52ec775e6b51c,tpl_9397997bc5d9,train,phishing,기타피싱 +93f249078062218842e3f2a265414aae8a613a919772769f8c3f429b7f5abef0,tpl_93f249078062,train,phishing,경조사사칭 +9549131fbaad3d165ffa4269912ab9ff988da0d74830956084cb21c43e9ecfad,tpl_9549131fbaad,train,normal,일상대화 +9666fb6f2231f0c120f41dd5428299d5f42f70fc045a33be35bf5e30829d9320,tpl_9666fb6f2231,train,phishing,정부공공기관사칭 +97f09c02f023e0daa013ee9ee125722f24f059a914166d67cf680c6ebc47f786,tpl_97f09c02f023,train,phishing,지인사칭 +bbdd90a53e90c4fdd866f55fd086ef03fbd275b706fa842a6295ea2811f589b8,tpl_97f09c02f023,train,phishing,지인사칭 +98c9faf79708ff8aa8ff582672fe59d531108cf1cc52cc955da5d0e4d424b7fc,tpl_98c9faf79708,train,phishing,경조사사칭 +9a17e496aea032ec4edc7355bfa89673f14a14bd017bec19bdc0c316cacd41b4,tpl_9a17e496aea0,train,phishing,기타피싱 +9a5486486b4b7767a8fa979f7095affef720529ee4bc714f8975dc25f1c0d19d,tpl_9a5486486b4b,train,normal,일상대화 +9a7703198c8425448ed96f03f9a7f30060a7c3d376eee64776e284a781bded93,tpl_9a7703198c84,train,normal,일상대화 +9acaf1de82c66fcf0ef9d30b4acb6791a766e992c224f9a41ec0693842b65b89,tpl_9acaf1de82c6,train,phishing,경조사사칭 +9ca4452fdfd46764ceb01d8118bc8e12f8ded6195c97aace09d63c06b2b8521d,tpl_9ca4452fdfd4,train,phishing,기타피싱 +9cd0a8fbce8f6e41f22b3ec11de1128c3f8adca9a9e7b75e67658895b590ff2a,tpl_9cd0a8fbce8f,train,phishing,정부공공기관사칭 +b8426f5e8c67eb9a82e09c6c9c52a1c1b63935a7944a57c84fcda7c393578ab8,tpl_9cd0a8fbce8f,train,phishing,정부공공기관사칭 +bec929960b9b614d52422c6eb39c7c514fdb558815c10ab295dd6e380c427838,tpl_9cd0a8fbce8f,train,phishing,정부공공기관사칭 +dedd15661ac4ac98e702afa8454c457bc5d88f1926cf67cfa8c09ca218b478f7,tpl_9cd0a8fbce8f,train,phishing,정부공공기관사칭 +f9d420f69dce5ec06362b2c9f8348b3924d911bb203a19bb53efff1113130dfb,tpl_9cd0a8fbce8f,train,phishing,정부공공기관사칭 +9cd9b89629cca2ecbc9a4a4ff693d719f75e60353649b1b55e88eeb6d6c2cce8,tpl_9cd9b89629cc,train,normal,정상알림톡 +9dcd508385eaacc630f2dd8e009a2e7205d0b01c7d33adb2ac78e4ef69ae82d7,tpl_9dcd508385ea,train,normal,일상대화 +9f13a10a2e223c7597ffdc38adc29a2498b51cc880f9717db757ec8d00128345,tpl_9f13a10a2e22,train,phishing,금융기관사칭 +a06248a1548c8d7ccf0fdd6bb4d21fa461deaa4b5adce94f4ebf286b81f86698,tpl_a06248a1548c,train,normal,일상대화 +a07e0d0ff8028e1ba3d5394e7c5383a7091cd0e952eb20e649de091114f4e492,tpl_a07e0d0ff802,train,normal,일상대화 +a196e1412916fbb36598f894179470f9caf254d5e48c2c3e8d380e0284e3d271,tpl_a196e1412916,train,phishing,경조사사칭 +a207d39ed33c2cfec0f6ad39ae952facc7201fe2b40b4b2fb91c366f2caa8351,tpl_a207d39ed33c,train,normal,정상알림톡 +a37afafc14fd1cc7025e30a8dfdb77ba4fe2e7b64f265f4ce9ce214586670d27,tpl_a37afafc14fd,train,phishing,경조사사칭 +a42ce53ef9911c934837c4d27d52ba69aa885a996b0127ce3bf3dbe0dc7afdc1,tpl_a42ce53ef991,train,normal,정상알림톡 +a4d87ea4bda6977cf26ed423973d7296a896a957f59c0382171184805b64e8a9,tpl_a4d87ea4bda6,train,phishing,경조사사칭 +a54becff10e9aa58d0049d16b63d2970b4194d9d2faa296a09a99f32977f17a4,tpl_a54becff10e9,train,phishing,경조사사칭 +a5bd16e9e8c638a07feac44cc91021ad0b5b5a56523b4f7a52050065625e0907,tpl_a5bd16e9e8c6,train,normal,정상알림톡 +a5ecc740dbdf8e6f9575082ed98a921b5d8e81b401816c7d0d6cc76c3e7d66db,tpl_a5ecc740dbdf,train,normal,정상알림톡 +a637a93dd231083a60d007f9548bd01513a7bbb493b865d6b529a4d267367c11,tpl_a637a93dd231,train,normal,일상대화 +a7fecec3955d55a85934f68d4184eec4933040aa80d8e9b9075653c307f7cbe6,tpl_a7fecec3955d,train,normal,정상알림톡 +a81efa689e5bab91577bbfd3c0ae3e32a941d3c8c7b18566cc15e6abf4479070,tpl_a81efa689e5b,train,phishing,기타피싱 +a864eafe5606bb28e25161425f85622df771fa89d63cea3bfc1a9203ee8cded8,tpl_a864eafe5606,train,normal,일상대화 +a877b904eafc181e79344e89b765f54d32c5855414f72d9aab7dc0481890bda5,tpl_a877b904eafc,train,normal,일상대화 +a88c399a59666a9830841bcc055685989d9a60369caeaa3572927dcd34dc6f56,tpl_a88c399a5966,train,phishing,정부공공기관사칭 +a8eea65ccc27637fe47071d1d32ffa49d7c3aa0dc00d7d7c6f339435979650c6,tpl_a8eea65ccc27,train,normal,정상알림톡 +a9632afda81cb033bd454b978abb2f533492c403cc34975e1bf6dbadad01ea4c,tpl_a9632afda81c,train,phishing,경조사사칭 +aa0d866a7d0183f97c3ec4889886fa15898bfcbae24279ea5d22578a7233ac85,tpl_aa0d866a7d01,train,normal,일상대화 +aa1e827b72b61a28ff1909e0a6bbd61ce081706b551fd8294b00dd4a1cb666a8,tpl_aa1e827b72b6,train,normal,일상대화 +aa3767924ec276839cc80113cc88cced1ead09cac5c54cf76298c9221358d2c8,tpl_aa3767924ec2,train,normal,정상알림톡 +ab6d79f90103d92ba46fa6c5e2c3d67ea18f411d9fe9adaa38b2ef24f29fd33b,tpl_ab6d79f90103,train,phishing,이벤트당첨사칭 +ab98406d8fcec61c853ccd8b1b530a0f9a8b82378f80683ec1b90431a5dd963a,tpl_ab98406d8fce,train,normal,일상대화 +aca082c34279159be8fc706ef9a75f9b0c5542abeeed3f674f94108f708ade76,tpl_aca082c34279,train,normal,일상대화 +acf056c5d8905517cd77259b0879f2aa91be6acd086d74286c5eda8ea7f62d1b,tpl_acf056c5d890,train,phishing,이벤트당첨사칭 +ad31612bdec487bbbf9452dc6ee419924311ff7e937744d1827bcd66bcd652ed,tpl_ad31612bdec4,train,normal,일상대화 +ad7ac004533e1ebd2fdb821362d434fa18662f056fd8c6c9cc6b9cf5d3ff8169,tpl_ad7ac004533e,train,phishing,경조사사칭 +ad942990feda8771c64d278166c14493ea842449d6ba34a29f0bebaf65c68c98,tpl_ad942990feda,train,normal,정상알림톡 +aeab5a99cd41d424a612aec8ff6d367a09bafc09efb99f2545039889cb2aa79f,tpl_aeab5a99cd41,train,phishing,금융기관사칭 +b043e482ac77fc3070d1dc5a783f1c23d643384d58ee73da24087ba8cec2c9be,tpl_b043e482ac77,train,phishing,택배사칭 +b19d139cec62222a2db8cbf003b6edb6ddab9246199ca989cf1b20d28f609774,tpl_b19d139cec62,train,phishing,경조사사칭 +b26a3e3b2577a322b081d8a796b4acdd45bc7aeec2308fb4b5d8c6a6b210c5bb,tpl_b26a3e3b2577,train,normal,일상대화 +b30fcac2bd7b1c92afad68c693c7a786a4d4a9a22533ca9ace6aa5ada28c5b44,tpl_b30fcac2bd7b,train,phishing,기타피싱 +b361070108c2a78800fe0d290b43efafb31827476b6cdd0df2dd2e48a947b8dc,tpl_b361070108c2,train,phishing,이벤트당첨사칭 +b383039194ff26581526cc37cd6428bbae23afaefcb3a3063d146af908d4ad20,tpl_b383039194ff,train,normal,정상알림톡 +b3afc430824d74ac0d38fdc229cabbc6f3cd40c999b80956be17e8959626e01e,tpl_b3afc430824d,train,phishing,경조사사칭 +b3eb96bd11080afe81f9be6801820cd487abd4332cb8ee085f678963d7dd080f,tpl_b3eb96bd1108,train,normal,일상대화 +b43e88b94d86e1f23dc84b139eea1382356a9bc825039efff1ecf05495a255a5,tpl_b43e88b94d86,train,normal,일상대화 +b4b02e3bff22d06209da4b727844fe6f8e4befb14f105b7dda00616cc65a16f8,tpl_b4b02e3bff22,train,phishing,기타피싱 +b55a703050774022d1a59436fc7b416b2739519ff4fa9b430d4993d6b55596b3,tpl_b55a70305077,train,normal,일상대화 +b69b277ba38103d07215834dea7fe4c4dc22b263ff19287acefaed9251910ff9,tpl_b69b277ba381,train,normal,일상대화 +b8c499e6a27fd1f5a192bc3a249cd60fad7036afae9cd1fab3815c03901ea18e,tpl_b8c499e6a27f,train,normal,정상알림톡 +b90debee503c6ca3751e2f30e22e6545cd2809fd8eb4c110972481e0032b1198,tpl_b90debee503c,train,normal,일상대화 +b9246644a76e55042f4a341160f94c23cca41af7ad091b7b2c20b2b34ae5d279,tpl_b9246644a76e,train,phishing,경조사사칭 +b93feee0fa35ab4e4d59776d615c8c46daf29cef2db518610430433e2dbe07e7,tpl_b93feee0fa35,train,normal,정상알림톡 +bacf00ec8cdc3f7d63ab95a58dc5fcd45fa6e2505a225a870f52ba9875c4dc74,tpl_bacf00ec8cdc,train,normal,일상대화 +bbdeb2a98386ef47f613da2fd2b057a3a99cb0d599f270453df4f12601968102,tpl_bbdeb2a98386,train,phishing,이벤트당첨사칭 +bc41b16c60ee1208a19a18427f538a175d2da534e0e5844d44dab3804285fb6a,tpl_bc41b16c60ee,train,normal,정상알림톡 +bc78fe06f89375357c6ff9a30af31a4ad18a46f008c68cf9f7d2d283d5444a88,tpl_bc78fe06f893,train,normal,일상대화 +bce074432ea37f4b3a0bfd41ca2dbd8a671dc1e27a28a19ae6594c31b2af8d44,tpl_bce074432ea3,train,normal,정상알림톡 +bd4e89ccd4369fdfd96e9107c0dc3b5f4686cad7b71bd04b70fc0e4427aafac1,tpl_bd4e89ccd436,train,normal,일상대화 +be01ed670804531fbfdbd3366fbe3599393d3d917da738c70758cff3dfeabe2a,tpl_be01ed670804,train,phishing,경조사사칭 +bee546a05e6ba85939c6e07c7e4712d293b98dd46a6f12785d2a67d7447c1d91,tpl_bee546a05e6b,train,phishing,택배사칭 +c00c42d5d62d57f7e55009c957691f3157cb5d53ae87975074e4d80f06b73ece,tpl_c00c42d5d62d,train,phishing,금융기관사칭 +fe75550baf0c6aa912efe098b994c0031913bbe155992368aea0d6801127775f,tpl_c00c42d5d62d,train,phishing,금융기관사칭 +c1815dc3d94d6cef446b868eb215dbbad095704e7ac9af4d852b0e1a32ee36d4,tpl_c1815dc3d94d,train,normal,일상대화 +c1e836f7a469c600d31e06487b5f9fc824d12aed3251576200fb8acfe49e1782,tpl_c1e836f7a469,train,phishing,경조사사칭 +c20223852c3b87ea26437199c357831b2a301cb3dff9ed149b19010087248529,tpl_c20223852c3b,train,phishing,경조사사칭 +c2c90aec6b8691fa77c0cd637c92c75fc4023ce987e471fbded32df5a8e1dcd4,tpl_c2c90aec6b86,train,normal,정상알림톡 +c31f0b91219609353a7cdd8c591715e6ca63af8ec478dd609f2d2aa37551d906,tpl_c31f0b912196,train,phishing,경조사사칭 +c3fac17c54146e6a0b7f40222a81398419741ad47a2278d3510fd67e5605191d,tpl_c3fac17c5414,train,normal,일상대화 +c43bfdf66bfb81943645f423ac23ed48680d71e12a2eb380a64e2b1e2a350c51,tpl_c43bfdf66bfb,train,normal,정상알림톡 +c4cc3cc9180aaf5e13a6b0f4807eb637afee02a6675ed00df3e5dd2ce8205dec,tpl_c4cc3cc9180a,train,phishing,금융기관사칭 +c52eff610ce203feff56c56602d8631d39f6ed37ca6fa51ea8a59713b228950e,tpl_c52eff610ce2,train,normal,일상대화 +c734846e5546b5b3a25550d096f90e789f7ed2839257ae785d22efbc485eae8c,tpl_c734846e5546,train,normal,일상대화 +c897d8bda0bdfc4b307c436f24d2d6ae846dcc069e88b9c82638e833325c5dd9,tpl_c897d8bda0bd,train,normal,일상대화 +c8d52f4ad56d8141cd7ce120a5dc1e975b865d45d9859b0d0249166e4e64e47b,tpl_c8d52f4ad56d,train,phishing,경조사사칭 +c93f5a36821c5c2b8da55869897655034cdcf78c17cb2d7a4a935933f67214a3,tpl_c93f5a36821c,train,normal,일상대화 +ca2cc85429e4f60699f865d1d80b5c708f28279ee57bb2a21303dad7d10bfa37,tpl_ca2cc85429e4,train,normal,일상대화 +cc8d0bb2a49bd34a471c0d7bd88040ca0d3fbab90e0c5a420acca2c9ac3366fa,tpl_cc8d0bb2a49b,train,normal,정상알림톡 +cd0c2b6cbc864c627df6e21514fe9724f712e8bbb8101ab4606775b5c12255cd,tpl_cd0c2b6cbc86,train,phishing,경조사사칭 +cef2488688bdd01e8fb5b577f6d93e1c85c2f936f2ed6afeaccfb847fe016aa2,tpl_cef2488688bd,train,normal,일상대화 +cf3a5f447f417e56a6acd365134a427a10ac98f7a9b2566b55ef7ff3a82b8cb4,tpl_cf3a5f447f41,train,phishing,금융기관사칭 +cf796f4eea0ff277c62bcce5e8ba36d3248f939616a5337eb01c651a9854ce36,tpl_cf796f4eea0f,train,normal,일상대화 +cfca46486c6a987552b9660e08f053a59082144d93d29e50375917b7f633c3d9,tpl_cfca46486c6a,train,normal,일상대화 +d130646e392118b03408e89cdb6567ac5ebc5ef282b5bc44782e77a196c9093c,tpl_d130646e3921,train,normal,정상알림톡 +d195f924160e219faa16a1835ecce9b47e36bbac4963d6b30d413ee2c3b5b0c9,tpl_d195f924160e,train,phishing,금융기관사칭 +d1bb21a25257bc2ba47c7609ebc43cd17be5abf0c32f4b49bec4cf987ec93f88,tpl_d1bb21a25257,train,phishing,경조사사칭 +d29974be811f288dfd78fd6ce6df5cff9b56b31f77a9a50a9da75a8910e526b5,tpl_d29974be811f,train,phishing,금융기관사칭 +d31bc0c92f3584a6608cfab969a655783fbff3b0cfc232d8b261fbce06872195,tpl_d31bc0c92f35,train,normal,일상대화 +d372e8000a42cd9af3d4c8f7ca31d411745f064821a94546a9c2ef9c33ad27ae,tpl_d372e8000a42,train,normal,정상알림톡 +d3f22e8b22090654088da1a3bc4d5685eb9bb589888b320593d86babdf0bb133,tpl_d3f22e8b2209,train,normal,정상알림톡 +d439a88e3eb1275d42d2cf416d066dc96f8fb474320595aa40ba58655ee94a6d,tpl_d439a88e3eb1,train,phishing,지인사칭 +d4b4642e7be22cf03a76281f286b38811b614281340be7aa09d5e7940a2dc8d0,tpl_d4b4642e7be2,train,normal,일상대화 +d5b9a5c7d227f2f6a0dbe6d268581389963019893ce9564553a310334234586f,tpl_d5b9a5c7d227,train,phishing,금융기관사칭 +d632e1f1594d3a51989bc45058cbe1d86d25ba776d6cfddf0975127cacdeb23d,tpl_d632e1f1594d,train,phishing,기타피싱 +d737e2950c234531e4e30aa2cb021fb0fc1d0cb49c746cad29242093f8172c9f,tpl_d737e2950c23,train,normal,정상알림톡 +d87d8668d828ff24a01a68de28c96b0c0fa384ceb6fe5819de16b50fe9647d17,tpl_d87d8668d828,train,phishing,금융기관사칭 +d8b87294730c72da113cdba5e9248674805c582c7940070e42f29e85b64c1333,tpl_d8b87294730c,train,phishing,기타피싱 +d9b63a059668853eeca89ff6e80675dd225af905948f011fc0c06aedf995475c,tpl_d9b63a059668,train,phishing,이벤트당첨사칭 +da710177852ec500cc5ff5a6a0b013bda4197f096dc153ea29850489f50fa979,tpl_da710177852e,train,phishing,경조사사칭 +db048dd893aa9e801ccb48204b85e7376337220151a7c489523b40b90d470490,tpl_db048dd893aa,train,phishing,기타피싱 +db6fa1e8200183d485a1386d0d097415a832912f89d9c6c9079a9c91020e4955,tpl_db6fa1e82001,train,phishing,지인사칭 +dba17dad83434ad598b373393296622135207f477012436be4ff1262f58c6429,tpl_dba17dad8343,train,normal,일상대화 +dc587a815be6d7c0af4fe3254349f210d99191c928d12d16decd550be75ae381,tpl_dc587a815be6,train,normal,정상알림톡 +dccd338f65a224e53bdcebddcfb764429ba4c928900b61298cc08fc2d63439f8,tpl_dccd338f65a2,train,phishing,기타피싱 +dda16189ab373bbda444f3661e711bf570e89dfd7dbe2ebe9d8e4a5ccfeb53f5,tpl_dda16189ab37,train,phishing,경조사사칭 +de2534ab6d414373b600e9742788c1895cb739a33b58500e0c2f04fbc4e13f5d,tpl_de2534ab6d41,train,normal,일상대화 +de3bf04c5568aeb13e43a1a5935fe911bce259b504701ee33b20cc70b553d7ad,tpl_de3bf04c5568,train,phishing,이벤트당첨사칭 +de61b396dddf9a3fb3ae3230dc1082f7d1032de9fbcb4d5c05d5505d07465d76,tpl_de61b396dddf,train,normal,일상대화 +decbd26769f4dcde0dad039cd04c2de4bdaf87e3e18d38b851316eef10c87b9f,tpl_decbd26769f4,train,phishing,경조사사칭 +e0216e7aad8b3584b4dd80064060472d6d4a8ae25da7fd33582fb8c11b77933c,tpl_e0216e7aad8b,train,phishing,경조사사칭 +e17e4f59d567ce23152dac10163a6a14552845eb21208885acf3f0a90cd71f66,tpl_e17e4f59d567,train,phishing,금융기관사칭 +e1dd61e8f30da5722021c4f3da82bdadfcc56271431f108bb0f59a932aa15430,tpl_e1dd61e8f30d,train,phishing,금융기관사칭 +e3cdc525e2bcc2109417f114a904f36ea4c947793a66b1e61b23f40ead82bbfc,tpl_e3cdc525e2bc,train,phishing,정부공공기관사칭 +e587c0bb7265dc3b9cb9beb4c87e7bb04254825b6f1b64028657a339a2e1e82b,tpl_e587c0bb7265,train,phishing,경조사사칭 +e5c9dc6c87df6253ad51eb437541d9a043735801226748efd01e14f6bcf4d096,tpl_e5c9dc6c87df,train,phishing,경조사사칭 +e5fb393c7657057a6a6f70130cec29f76185e57bd39de611528a178f5ea71d99,tpl_e5fb393c7657,train,normal,일상대화 +e68dc3228107c728df275478de2c35da3c26a96ea5c1b2c24cfd9fe2186c4440,tpl_e68dc3228107,train,phishing,정부공공기관사칭 +e6ae0b7e57b44abf9b43e9bcf70cd888754002d089c1d8439239883a04f0a953,tpl_e6ae0b7e57b4,train,normal,일상대화 +e6cd21c9e01cb99e84865edc5b5bab1377bb48a1feb147298d81fae4c750e12c,tpl_e6cd21c9e01c,train,normal,일상대화 +e748c7274d6ce8ee555f9f8cd5e86a67927ef282c831ed5ace423f7e7a7a4cc1,tpl_e748c7274d6c,train,phishing,지인사칭 +e74c1d56ba58a67391154c313cd91ff2bed0e8b9f489639dce5d86614e5ae32c,tpl_e74c1d56ba58,train,normal,정상알림톡 +e7bbc2b6458f528b331b9c360cd671f730f8c4db3dc8dce960be50fad35a5117,tpl_e7bbc2b6458f,train,normal,정상알림톡 +e7d9b4f2986c4bc38f47a276d2b82996b6ce1ec324d9dda44a97ffafd3ae7064,tpl_e7d9b4f2986c,train,normal,정상알림톡 +e8eecdce959899fc63e438af29527aa46d5ac383e8c6e563668612638af95be3,tpl_e8eecdce9598,train,normal,일상대화 +e9b4563b8a4744dcf6fecb84f797914ba0b418f4e448e68b414a8ad6a3014c05,tpl_e9b4563b8a47,train,normal,정상알림톡 +e9d2083bb674dba6d0aa23d40ecc564e092c657dd532a4a5a25d6b5c33a422f2,tpl_e9d2083bb674,train,normal,일상대화 +ea2c9cd83cc9793648fb909ef2f1979a265b3f4c22ead4575cd7dabde646c61e,tpl_ea2c9cd83cc9,train,normal,일상대화 +ea3ab3b2b80c54f98d013e6a512550d3f4200631eca89d6b91ba1cd195c41d02,tpl_ea3ab3b2b80c,train,normal,일상대화 +eb4e2af0a87e2dbb9e8e35fff321c7f542cfb840aabbf8365ec6390b6593b959,tpl_eb4e2af0a87e,train,normal,정상알림톡 +ecc2f1fb87b4f69bec598ba7082cbbd47fc6651dcbf29c08333e5aefe4666c4e,tpl_ecc2f1fb87b4,train,phishing,금융기관사칭 +ece548fb0c416228c3161bc75fa52c2799cbc3973f2578dba4164f5299dc803f,tpl_ece548fb0c41,train,phishing,경조사사칭 +ecf555c7d3e0f1564a612890874224958e2c9550ce381e05de89216180406b5a,tpl_ecf555c7d3e0,train,normal,정상알림톡 +edbc32617ba53e87dc27ccadb713e7b754ad3762050081fac8270bdd3d4a0b4c,tpl_edbc32617ba5,train,normal,정상알림톡 +ef5f8a366f8045c674c00d6ee22a05c8c16a27f83908aac1f17ed6765e0dfa12,tpl_ef5f8a366f80,train,normal,일상대화 +f0c587f4a72d20b051749881933950724532bd39f11e5df48531d1203ce6e778,tpl_f0c587f4a72d,train,phishing,경조사사칭 +f107f66e30f5ae10417b340f5821f8fd717415b6ef902ff9071c91b7b1ad233d,tpl_f107f66e30f5,train,phishing,이벤트당첨사칭 +f1b5a5ffbdfb2d29d3d548a54a3730b85a78f34722e509c51a04d7f87540dc04,tpl_f1b5a5ffbdfb,train,normal,일상대화 +f1ed0107d7276167b29a9aaf1372646c6762967790f5c41445f0b89ee8d31c10,tpl_f1ed0107d727,train,phishing,기타피싱 +f23eb0d0413c75d0f1b5bcc95a12777bb2f0d432a9389fa8e7251ce6f2d29ccf,tpl_f23eb0d0413c,train,phishing,기타피싱 +f24bd95a0105856c69849469a6eb5fabb51201dd16f66ff9c0d086b32fac2714,tpl_f24bd95a0105,train,normal,정상알림톡 +f333e9d765c7af70d483fbea5a628bef2fd81b2a8f9b93c4cd660334769ed98c,tpl_f333e9d765c7,train,phishing,이벤트당첨사칭 +f34ff6d881cf2d768266c08310e10901540eddd6ee55bb02a06dacf675320ae2,tpl_f34ff6d881cf,train,normal,일상대화 +f360149d7eee2f4f6bbe68d19ddb2031fc5a119971ea3484ae1338810ff9afe7,tpl_f360149d7eee,train,normal,정상알림톡 +f5632aa15a179afb0a9bda37a4f4872d0507a8d1cb5b9b194313f5b0e4164453,tpl_f5632aa15a17,train,normal,정상알림톡 +f89ab6bcd7f1b5c4747133c97545d83ea3ab579426925982e84caba01971fa68,tpl_f89ab6bcd7f1,train,normal,일상대화 +f8e9d60f0fe364e7d1aaf27aaa72935ab6be4a3533c9161e5872cfb98babd911,tpl_f8e9d60f0fe3,train,phishing,이벤트당첨사칭 +f959065a3d9eda0e3a949ae8344ee4f67cffb7e52564a0c36d72d1c0897409e9,tpl_f959065a3d9e,train,phishing,택배사칭 +fa4ca692d5588d84f1af7e44e76db19627f619b52b6ae638a809c4691f118568,tpl_fa4ca692d558,train,normal,일상대화 +fa9a518b8e529a6cf3ba22c869764ace7ba493fe109251e79b61ced209c73e52,tpl_fa9a518b8e52,train,normal,일상대화 +faa6665d01c251c84d1d651c5ba9691acfce5dffda65d454f249d36c80991263,tpl_faa6665d01c2,train,phishing,이벤트당첨사칭 +fb62fd639af13c2aa2e7cfc6de16ba6759cca9b6bd1a9ca114107643ad2dcdf9,tpl_fb62fd639af1,train,phishing,정부공공기관사칭 +fc21a890b8b458eabc6da99c2cec29428e8c49cbfe2f3a0e32c676f7a188ba6f,tpl_fc21a890b8b4,train,phishing,경조사사칭 +fca1b9cc71bff612b9877ae03e79de1cfa22ab88fd8ab3958d9f2daac22e5cae,tpl_fca1b9cc71bf,train,normal,정상알림톡 +fcf8ff285f4e52bdd8d63d27f561228241b6261dac2cdc331f547182fdee83f4,tpl_fcf8ff285f4e,train,normal,정상알림톡 +fe00ed6c2ed234e40809cba49171dea1633ec3f89d7ac15244c5d481c6e52c0a,tpl_fe00ed6c2ed2,train,normal,정상알림톡 +03ce45d2245f226d067ab61df8781c8ca180b16fefc6e2780127d609b0280ec4,tpl_03ce45d2245f,validation,normal,일상대화 +040d8bc48601544c17715c0519b54fd7f13301d95ccb22521afe0a07d28e114a,tpl_040d8bc48601,validation,normal,일상대화 +09555dd484584de2b36b5bb7aad6ce432ad88655f7a49def3059f9cd896e5ceb,tpl_09555dd48458,validation,normal,정상알림톡 +0a65faace206d49611fdf4bbedf85a46cedfc1951a29fd65aed60c4dd65bfa65,tpl_0a65faace206,validation,phishing,택배사칭 +3ed7794ebb802df7838733a7d99314716c649f23755a9e2a78ca70dea9800739,tpl_0a65faace206,validation,phishing,택배사칭 +bd97974056780e2feec98022113ef1d514a1eb5a68f9c636d0abb9a4568ef6e7,tpl_0a65faace206,validation,phishing,택배사칭 +0a8b0b7eaef3845e40edf3c22c3c6d17700ff066f1b6cd1bef7b8f24779e46dd,tpl_0a8b0b7eaef3,validation,phishing,경조사사칭 +0c4eec4898c345f4c4c9ba05723ead5c0c261c6d7e867fedcbbe4304eda3f806,tpl_0c4eec4898c3,validation,phishing,택배사칭 +75b5e35b3a4b942e500b0e89b183e21c2f698e026879f5cf6b6421883ed68738,tpl_0c4eec4898c3,validation,phishing,택배사칭 +0e15d08e06993b66e2e50ddf41e0d431bf5d5f4a18d44774e95c3a38c8cbe35e,tpl_0e15d08e0699,validation,phishing,지인사칭 +15e3408fe377c99e7b7d3c1ebd0f20551abe2c63b72487ccb2e734b009a2e64c,tpl_15e3408fe377,validation,normal,정상알림톡 +19083a33b3ca8bf82d76f419e0d5eb17bb680173efe693a84f0b1ac3b76e6185,tpl_19083a33b3ca,validation,phishing,기타피싱 +23424e2f1ddd6dbc3461927c36647ebe8d05e1f07a983cab157e57922c66576e,tpl_19083a33b3ca,validation,phishing,기타피싱 +289b21983f4d981cd429323483365d60e967d089656d930db86180446bbc44f0,tpl_19083a33b3ca,validation,phishing,기타피싱 +6258927cab37de77b0e9c595655550c6facb3554a48bd4a6a93ae51c941f948d,tpl_19083a33b3ca,validation,phishing,기타피싱 +62b95e80ee4df87bef700d50353935ec0c03b55cb757a477f7b618af76a4a929,tpl_19083a33b3ca,validation,phishing,기타피싱 +63a5cd7a71b274ed486df1231620f022411f415a17d4626be0d06c3a4b413ad3,tpl_19083a33b3ca,validation,phishing,기타피싱 +68b6d0932336643154fbc615fe5ba1931507034f2a43af035ad874105292a013,tpl_19083a33b3ca,validation,phishing,기타피싱 +6daa5a4aff6686745dadf7610d8bf37af6ff25b9744674e9612c309e2fc1dccd,tpl_19083a33b3ca,validation,phishing,기타피싱 +8487df167a07d9616fae22660e3a53e4e66bc2ed7ab42b0e51bfa3f85cce9305,tpl_19083a33b3ca,validation,phishing,기타피싱 +84a4fe8123578e2c2e0d1820e60f1ca9a0e34edcc6ee8a90a3b0247ae24b1cdb,tpl_19083a33b3ca,validation,phishing,기타피싱 +89e88e60e240c2950cd4b3c079330ec866e8a32a0c846a73321d502123315ff9,tpl_19083a33b3ca,validation,phishing,기타피싱 +a581cb487418137bf2fd2234f778a8484841261142231e9027fef2d26a182a68,tpl_19083a33b3ca,validation,phishing,기타피싱 +c24d1722fc36edbd26fd1e037c8a477a2267fa89cac88467fc445e9915c53707,tpl_19083a33b3ca,validation,phishing,기타피싱 +ce3bce25e58cd367838a3e4e966528f698d5111cc8fde025849d8bef453d380f,tpl_19083a33b3ca,validation,phishing,기타피싱 +cedbb48a7c18637e01c6b63242c86f842c1285f7e8547d5f4fab507c7d51cad9,tpl_19083a33b3ca,validation,phishing,기타피싱 +d24b4843f5f3541a8be37a7d3ccd800b119afc317628223bd25897bd5cf0267c,tpl_19083a33b3ca,validation,phishing,기타피싱 +fdc78ec73266220ad04f3f815414abfbae0ca45fb459b7307255d20795877431,tpl_19083a33b3ca,validation,phishing,기타피싱 +1e5406db2516f7b670add8e9b5f1b0433b75e7274d1aab51dd686f8f9fda21ec,tpl_1e5406db2516,validation,normal,정상알림톡 +20d5d4994a709e1482826653a1a1235fa785a146af86129bd5758f9fe2a89c5c,tpl_1e5406db2516,validation,normal,정상알림톡 +35a3c97e934d16756f8555c368a2d406e847c31b3614a98182b67d7ec789f7ae,tpl_1e5406db2516,validation,normal,정상알림톡 +828b133c682a90cf80dc66ddb563f0bad1add03d628f6ebb70e813269eba681f,tpl_1e5406db2516,validation,normal,정상알림톡 +85b54d4b743b7c9aa944f207dcd94d53a45dd83a54e86a9ba4eb25ee7bc61dd1,tpl_1e5406db2516,validation,normal,정상알림톡 +b2b72d199febcdff9faa65692b019e5ed10189500644fcc8b3a1051a937b92f3,tpl_1e5406db2516,validation,normal,정상알림톡 +fd197550001176f4f92c28379b3e0dd6e0fe9a636a00802efeeced7bceb80167,tpl_1e5406db2516,validation,normal,정상알림톡 +2387227ed26ae9249a4a4fc187028fdcc7b7a4e397edec018364674d0a816ff5,tpl_2387227ed26a,validation,phishing,경조사사칭 +27306066a0371a45533d6217c0c4c36a64678d736cf406d90c9be89ff77afe14,tpl_27306066a037,validation,normal,정상알림톡 +279f61be1f75cf521897e86a237590d1d287b833dcf8364971b41d4f82ea40f7,tpl_279f61be1f75,validation,phishing,금융기관사칭 +4146fe324ad837d03bd7af52dc292b6546cc64e3382ca425483f8a0b23505ca8,tpl_279f61be1f75,validation,phishing,금융기관사칭 +461cda61772f816922fc24a6e79db8c0d942e3747eba7a4c8356842298ef03f6,tpl_279f61be1f75,validation,phishing,금융기관사칭 +54c7f65b612ca668bc04d3149ff8d829f08ba3f4a8291d38557340422432f197,tpl_279f61be1f75,validation,phishing,금융기관사칭 +591d1d3671d7ddba2e5739d9a73359500dd3d8c2b416fc625513db77945d51d6,tpl_279f61be1f75,validation,phishing,금융기관사칭 +5ee3eac81b36eadf0badc4a7499e89d5dc2b340f76eb7d44836b859514fa2a73,tpl_279f61be1f75,validation,phishing,금융기관사칭 +aaddf84eb21b5ec684a769f73db5aaf99c4096826cef79cf04599a5d8d69e74e,tpl_279f61be1f75,validation,phishing,금융기관사칭 +b5e1883331ca8d274bf06474d5078ed3086127e15e9c449f74e09dca06104e58,tpl_279f61be1f75,validation,phishing,금융기관사칭 +bb6579844879096b79d10f1b04f46209cf9c1d895627aee57800c468341f7686,tpl_279f61be1f75,validation,phishing,금융기관사칭 +bf56cdc7392e5da03fa2ac37b7c88f967f78316faa115ce6e7b112b1a61e83f8,tpl_279f61be1f75,validation,phishing,금융기관사칭 +cc5dae5f1cb4a5365e3b7c5a3adcab875e9bdf01d34368407968ce45073df0e7,tpl_279f61be1f75,validation,phishing,금융기관사칭 +cf1ef4789f675d47f01afb1e54ac37f6682556f93dc861b0b1f2d8eef327cd29,tpl_279f61be1f75,validation,phishing,금융기관사칭 +ee9ea5e201ddcfbfb677f0a281d4e2dceb2a1bb22274de4a9ea556ab6102a068,tpl_279f61be1f75,validation,phishing,금융기관사칭 +2836706edb74f93eaf81cda99e04d8c593f07e6f440a8429c89a8ece1558a330,tpl_2836706edb74,validation,normal,정상알림톡 +2a54df5502fd7bb5be95be59a976ea4a6d590e48ebbe4529b92f61f1b857bbf7,tpl_2a54df5502fd,validation,phishing,경조사사칭 +2a64f6dff869c1e3f0d854db4f982650f9c170cfc3df6d41186dbc59a37390df,tpl_2a64f6dff869,validation,phishing,택배사칭 +dd1953163d90978c0af2e292d478462f41a4f7afa4ec6d1111dadd5490c2afaf,tpl_2a64f6dff869,validation,phishing,택배사칭 +2ea8346732eddecebb3d5b0af2dadfe19eac847d63d1a54616e3c9780c0d0a9b,tpl_2ea8346732ed,validation,phishing,기타피싱 +2fa8cd4ab036247743cf41c88548580b91e22ee723c2999aad1a144eeb6d46df,tpl_2fa8cd4ab036,validation,normal,일상대화 +301f545f30fe3a4b2ea8bf0be80d564782893f0137f7fc10645244a3d82eb833,tpl_301f545f30fe,validation,normal,정상알림톡 +3256ba9d4ad81438c3a78ad2d9d9124a2f6db7d2ba33c934f2093f1ea2b55742,tpl_3256ba9d4ad8,validation,phishing,금융기관사칭 +338a125a2c7e9bac3aa623fc0d4a5c57fad9eabe480ac9dedfb1b9c53ffcdb31,tpl_338a125a2c7e,validation,normal,정상알림톡 +354a654dd179372d1c17ae688d9cffbc5ff4afaddf944bffe01a178152368903,tpl_354a654dd179,validation,normal,정상알림톡 +35588584272193b1d537bf8a49377e3e7de4b33858e185146dbbcb557163a375,tpl_355885842721,validation,phishing,이벤트당첨사칭 +37474d08d7062e7a6f3d7419419b286d2eff25617e96a9f3e1f2d06cf5858588,tpl_37474d08d706,validation,normal,일상대화 +38e8b37a3a7748541b8dff60ca5f4ff25b9e09c9869e3781d376249e1106fdc2,tpl_38e8b37a3a77,validation,normal,일상대화 +3e6d4cfe2fc8f1cce1518622632ea2f3fb5998fd42ee64414e27d84c85f0f35e,tpl_3e6d4cfe2fc8,validation,normal,일상대화 +3ee443e030ab5aff20c11eb1d2a3857f363d8fe4a989e99448b0e12e9069e216,tpl_3ee443e030ab,validation,normal,정상알림톡 +3fbc176696780069a098798320728ee62b57c2b996ecacc6c9a4edad4e37adc3,tpl_3fbc17669678,validation,phishing,이벤트당첨사칭 +4142fcb037bf6abb39e3c8d499beba8b3f08276d63f3f041911c1ebd7a574222,tpl_4142fcb037bf,validation,normal,정상알림톡 +44ac70c22435964a956e8332fbac2a1a4f66d2c0d7cb8a99f3c72ea159ff15fe,tpl_44ac70c22435,validation,normal,일상대화 +46c8caa872762ad9a7ed9a0205217fe0744867fc655e350a89a1d1e3935983eb,tpl_46c8caa87276,validation,normal,일상대화 +4c134bbea200a40d01e47912d3654b986521efabfb5db11f5c82543798af7223,tpl_4c134bbea200,validation,normal,일상대화 +4e30669a372ba70dc6b8c55174d73338ddfec668600f6bed404433217a3ea3e2,tpl_4e30669a372b,validation,phishing,금융기관사칭 +52b243ae5b2c5906b0f4edf3661eb1c72f176d74dfc9b8c8858fdd93929dae64,tpl_52b243ae5b2c,validation,normal,정상알림톡 +534c8fd679825d3fe1f006f271adb420465ef772ba31a9bcb5d8d10f0c8a2db7,tpl_534c8fd67982,validation,normal,정상알림톡 +5580c99edd3218684721ed377d478d39496755e1956427c69ec48ccedee48db8,tpl_5580c99edd32,validation,normal,정상알림톡 +5d9ba9d968cbdabd9b4195fa75f26fda3999f6975a0132673bc7f6f3166a03dd,tpl_5d9ba9d968cb,validation,normal,일상대화 +6024fd17f2fa6141365cda4947044270b50b780af40ad659739c652222768dc2,tpl_6024fd17f2fa,validation,phishing,택배사칭 +97f533f44175aac574811c5658326cb5400350964ee96e623c899575a5665cc4,tpl_6024fd17f2fa,validation,phishing,택배사칭 +60aced738636c31917e9dc613957ed3c0d5963a2aa417b94ab6cf430610891b1,tpl_60aced738636,validation,normal,일상대화 +62c92bf8de0a2c156699eadec10c64deb621600a03054e382243edfc2439635a,tpl_62c92bf8de0a,validation,normal,일상대화 +638d116d425486fdd6c6d02756dbc9f8c32288aa05c6129adbcd6ef8ee8e127c,tpl_638d116d4254,validation,phishing,기타피싱 +6474a1c063ddf8391fc73618c6cb3fdfa5080083dfb7bee9c5f65c67f46ddf3f,tpl_6474a1c063dd,validation,phishing,기타피싱 +6954d76bffa4df64d32eece72440e5da18f1f6546b2f98ec65b344bbdfac4056,tpl_6954d76bffa4,validation,phishing,금융기관사칭 +b9d87b03878c6d21e555baac6698d2371eecb7bbc3e4a0a0d6ca97640535e6cb,tpl_6954d76bffa4,validation,phishing,금융기관사칭 +6bb7f65cd1958ede8f5a8249b600917679b237e53da742ddf1b44c9b7de53da7,tpl_6bb7f65cd195,validation,phishing,경조사사칭 +6dc0226034f1ff47ab4b3951cbab80e7ec8f1f32bd9260259dec9066422239ad,tpl_6dc0226034f1,validation,phishing,금융기관사칭 +be368783e22c852fa68bb0b019860bdfecc281d023da26cfc09dc60dab0b1fb5,tpl_6dc0226034f1,validation,phishing,금융기관사칭 +70fc743b6c40fb7f712a1b968a2aed72b0b2086796ed236f2b955dce428807e0,tpl_70fc743b6c40,validation,normal,일상대화 +7243157b2c8f9fab1664cd2b01270de2d106b872c1620e2402a1c9329c2ba243,tpl_7243157b2c8f,validation,normal,정상알림톡 +80ec9a6262b41bb74fd4aad4ca37cb69bbf51f2a32d4b17c14fa4913ed0dffdf,tpl_80ec9a6262b4,validation,normal,일상대화 +8621a0e991ef94c147a956420fb536f57728641dc59ca84a7852b5becae4a6a4,tpl_8621a0e991ef,validation,normal,정상알림톡 +88414a67084984f916b409e3bc704bd8a39d2b98d954db54a3795da656b526af,tpl_88414a670849,validation,normal,일상대화 +8987a142dd017dfb47ca35df5a57f7e8129e17cc47a5017faf7581597a6043c6,tpl_8987a142dd01,validation,normal,정상알림톡 +956f6a3f1e97884c8ba70f0a41b6dc23cc8505b5d5a6c9fe8fbf50b55938194c,tpl_956f6a3f1e97,validation,normal,정상알림톡 +95717bed5eabd63b3290b6aa75dc43f9e7a08431f602dc3cd46f6ebf4f7b9952,tpl_95717bed5eab,validation,normal,일상대화 +9d74c49cbbace6ec43aa322f9cc08efe521cff81975099320b53e1a90a2de4a0,tpl_9d74c49cbbac,validation,phishing,이벤트당첨사칭 +af129827c3283f2218baef8752bf17dc48159f820b6793985bfcd7c83213f83b,tpl_af129827c328,validation,normal,정상알림톡 +b1c8be95c9ab6584c0b777203c854a975f01211ed0d120cd4d0ad91e8797188d,tpl_b1c8be95c9ab,validation,phishing,택배사칭 +b686a63621b386fe88f773d8c555db9530d1f98c33331ba0698a6f2210252549,tpl_b686a63621b3,validation,normal,정상알림톡 +ba268612dae4cff637f6a92ef7c0eacec6d0c43a1d9025319a0c25271800365a,tpl_ba268612dae4,validation,normal,일상대화 +bf441f2256a139b25ae45c3191c7c8104c8bf93415153aa2da91b9800b88bcc3,tpl_bf441f2256a1,validation,normal,일상대화 +c2cfdee4aca9f5222c5fea8cd933dd08fa4033d474d81a662e3c4d9c6d3e1a10,tpl_c2cfdee4aca9,validation,normal,정상알림톡 +c452cc059cd868f8971aae413710acde6b4c8177767bb8496694174d800d8d02,tpl_c452cc059cd8,validation,normal,정상알림톡 +c836271843951f99322b89fbec45ad5d727ef2ada3379025b62cbcfb3f4d614a,tpl_c83627184395,validation,normal,일상대화 +c92d09113954a46ddae4637ee2e6f3880e73253b44f87eaa895b1d0ed4908a3e,tpl_c92d09113954,validation,normal,일상대화 +d029b5db2339e492e3301b191e408fb479fbecb6639c0ddb77c314d3b7d3535b,tpl_d029b5db2339,validation,phishing,정부공공기관사칭 +d0314a36541c010bf2cdee6550d8b4302dd410024c4d5776853b4446e94e5903,tpl_d0314a36541c,validation,normal,정상알림톡 +d032c79c99bf3ca9213f7263f47f32cfd8526e911d5a849518b237e4517ad7ba,tpl_d032c79c99bf,validation,phishing,기타피싱 +d12d5c8c893c7770de85d622bb84933abd4c5b8dd6da76a497f7db729c9cb1ca,tpl_d12d5c8c893c,validation,normal,정상알림톡 +d5ef9fb314c88a5c4c493b37f59172796dbb71698c4584dae3d2869aab7d065d,tpl_d5ef9fb314c8,validation,phishing,금융기관사칭 +d9c7443406c09a14c47d147548e4adec2eacc9b756654b9362db7e0ea2e9bef6,tpl_d9c7443406c0,validation,normal,정상알림톡 +dc54db46b6aaee174c4010a54d14d160df29dae43373528c18d19f0636d7aefc,tpl_dc54db46b6aa,validation,normal,일상대화 +df8da366209dfcee42831f8793856cb846cc54a408f5a55bf71bb10b6f56a594,tpl_df8da366209d,validation,phishing,이벤트당첨사칭 +e5b594eb6a0a67c8933272f9959417bc713b30e6dd4d2ff156999d444d55243b,tpl_e5b594eb6a0a,validation,normal,일상대화 +e640f184e93171a29597e00f2ff842ac6429be9f8284a7ec937c36a0d4a09764,tpl_e640f184e931,validation,phishing,금융기관사칭 +ed4b5111d7e5b8c935a5242e1c1ed1910de62108e31e32396bc5030e1f5a76dc,tpl_e640f184e931,validation,phishing,금융기관사칭 +eb8f6d7e5925e119738ba91c9b0ed18f2ecb8d9b2a3a3bff7ae366435b3681d4,tpl_eb8f6d7e5925,validation,phishing,기타피싱 +ee03ab85bde5b1e60c128fbd34eabecc0c854d63f30131db8df9f8071ba28de8,tpl_ee03ab85bde5,validation,phishing,금융기관사칭 +f02abf0cbc9b039b5a6262b60f5f29d478c53e6a6d0bf51384a06f005a7ba373,tpl_f02abf0cbc9b,validation,phishing,이벤트당첨사칭 +f07fa7b32bd05fae33a1c80fa057a23a46876a485bd7ed55e85101e468ade71a,tpl_f07fa7b32bd0,validation,phishing,금융기관사칭 +f3162cc2b253bc20515e6f75b914ce103f69f9ed6dd5b2cf7aecbc8965368de7,tpl_f3162cc2b253,validation,normal,일상대화 +f6455850e59557736f114918a5550a75dec3de9b2e37a3b9c3d812438a5b0133,tpl_f6455850e595,validation,normal,일상대화 +f66a0fcef6a6e9308d83ba512940d123e74d0a11ed0756d542c64a30503da414,tpl_f66a0fcef6a6,validation,normal,일상대화 +fbb01efce1082196ce7ba664c1c16115bc0374a277c921c0d826091de84fe441,tpl_fbb01efce108,validation,normal,일상대화 diff --git a/data_science/SMSModel/template_grouping/__init__.py b/data_science/SMSModel/template_grouping/__init__.py new file mode 100644 index 0000000..381bdcb --- /dev/null +++ b/data_science/SMSModel/template_grouping/__init__.py @@ -0,0 +1,21 @@ +"""SMS 템플릿 그룹화의 공개 API.""" + +from .config import TemplateGroupingConfig +from .fingerprint import ( + add_text_fingerprints, + create_text_fingerprint, + remove_exact_duplicates, + validate_duplicate_labels, +) +from .service import prepare_template_groups +from .similarity import assign_template_groups + +__all__ = [ + "TemplateGroupingConfig", + "add_text_fingerprints", + "assign_template_groups", + "create_text_fingerprint", + "prepare_template_groups", + "remove_exact_duplicates", + "validate_duplicate_labels", +] diff --git a/data_science/SMSModel/template_grouping/config.py b/data_science/SMSModel/template_grouping/config.py new file mode 100644 index 0000000..0809832 --- /dev/null +++ b/data_science/SMSModel/template_grouping/config.py @@ -0,0 +1,37 @@ +"""SMS 템플릿 그룹화 설정""" + +from __future__ import annotations + +from dataclasses import dataclass + + +DEFAULT_SIMILARITY_THRESHOLD = 0.88 +DEFAULT_NGRAM_RANGE = (2, 5) +DEFAULT_MIN_DF = 1 +DEFAULT_MAX_FEATURES = 50_000 + + +@dataclass(frozen=True) +class TemplateGroupingConfig: + """동일한 입력과 설정으로 그룹화를 재현하기 위한 불변 설정""" + + similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD + ngram_range: tuple[int, int] = DEFAULT_NGRAM_RANGE + min_df: int = DEFAULT_MIN_DF + max_features: int | None = DEFAULT_MAX_FEATURES + + def __post_init__(self) -> None: + if not 0.0 < self.similarity_threshold <= 1.0: + raise ValueError( + "similarity_threshold must be greater than 0 and at most 1" + ) + + min_n, max_n = self.ngram_range + if min_n <= 0 or max_n < min_n: + raise ValueError( + "ngram_range must contain positive values in ascending order" + ) + if self.min_df <= 0: + raise ValueError("min_df must be greater than 0") + if self.max_features is not None and self.max_features <= 0: + raise ValueError("max_features must be greater than 0") diff --git a/data_science/SMSModel/template_grouping/fingerprint.py b/data_science/SMSModel/template_grouping/fingerprint.py new file mode 100644 index 0000000..5d4fec9 --- /dev/null +++ b/data_science/SMSModel/template_grouping/fingerprint.py @@ -0,0 +1,78 @@ +"""정규화 SMS의 fingerprint 생성과 완전 중복 처리""" + +from __future__ import annotations + +import hashlib + +import pandas as pd + + +def create_text_fingerprint(text_norm: str) -> str: + """정규화 텍스트에서 결정적인 SHA-256 fingerprint를 생성""" + if not isinstance(text_norm, str): + raise TypeError("text_norm must be a string") + + canonical_text = text_norm.strip() + return hashlib.sha256(canonical_text.encode("utf-8")).hexdigest() + + +def add_text_fingerprints( + df: pd.DataFrame, + *, + text_column: str = "text_norm", +) -> pd.DataFrame: + """원본을 변경하지 않고 text_fingerprint 컬럼 추가""" + if text_column not in df.columns: + raise ValueError(f"missing text column: {text_column}") + + result = df.copy() + result["text_fingerprint"] = result[text_column].apply( + create_text_fingerprint + ) + return result + + +def validate_duplicate_labels( + df: pd.DataFrame, + *, + fingerprint_column: str = "text_fingerprint", + label_column: str = "label", +) -> None: + """동일한 정규화 메시지에 서로 다른 label이 있으면 중단""" + required_columns = {fingerprint_column, label_column} + missing = required_columns - set(df.columns) + if missing: + raise ValueError(f"missing columns: {missing}") + + label_counts = df.groupby(fingerprint_column)[label_column].nunique() + conflicting = label_counts[label_counts > 1].index + if conflicting.empty: + return + + example_columns = [fingerprint_column, label_column] + if "text_norm" in df.columns: + example_columns.append("text_norm") + examples = ( + df[df[fingerprint_column].isin(conflicting)][example_columns] + .head(10) + .to_dict(orient="records") + ) + raise ValueError( + "identical normalized messages contain conflicting labels: " + f"{examples}" + ) + + +def remove_exact_duplicates( + df: pd.DataFrame, + *, + fingerprint_column: str = "text_fingerprint", +) -> pd.DataFrame: + """동일 fingerprint 중 첫 번째 행만 남김""" + if fingerprint_column not in df.columns: + raise ValueError(f"missing fingerprint column: {fingerprint_column}") + + return ( + df.drop_duplicates(subset=fingerprint_column, keep="first") + .reset_index(drop=True) + ) diff --git a/data_science/SMSModel/template_grouping/service.py b/data_science/SMSModel/template_grouping/service.py new file mode 100644 index 0000000..e689617 --- /dev/null +++ b/data_science/SMSModel/template_grouping/service.py @@ -0,0 +1,38 @@ +"""SMS template grouping 처리 순서를 조율하는 공개 서비스""" + +from __future__ import annotations + +import pandas as pd + +from .config import TemplateGroupingConfig +from .fingerprint import ( + add_text_fingerprints, + remove_exact_duplicates, + validate_duplicate_labels, +) +from .similarity import assign_template_groups + + +def prepare_template_groups( + df: pd.DataFrame, + *, + config: TemplateGroupingConfig | None = None, + text_column: str = "text_norm", + label_column: str = "label", +) -> pd.DataFrame: + """fingerprint, 충돌 검사, 중복 제거, 유사 그룹화를 순서대로 수행""" + result = add_text_fingerprints(df, text_column=text_column) + validate_duplicate_labels( + result, + fingerprint_column="text_fingerprint", + label_column=label_column, + ) + result = remove_exact_duplicates( + result, + fingerprint_column="text_fingerprint", + ) + return assign_template_groups( + result, + config=config, + text_column=text_column, + ) diff --git a/data_science/SMSModel/template_grouping/similarity.py b/data_science/SMSModel/template_grouping/similarity.py new file mode 100644 index 0000000..129c6c7 --- /dev/null +++ b/data_science/SMSModel/template_grouping/similarity.py @@ -0,0 +1,146 @@ +"""문자 n-gram cosine similarity 기반 SMS 템플릿 연결""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.neighbors import NearestNeighbors + +from .config import TemplateGroupingConfig +from .fingerprint import add_text_fingerprints + + +GROUP_ID_HASH_LENGTH = 12 + + +class UnionFind: + """유사 메시지 쌍의 연결 요소를 하나의 그룹으로 합침""" + + def __init__(self, size: int) -> None: + self._parent = list(range(size)) + self._rank = [0] * size + + def find(self, item: int) -> int: + if self._parent[item] != item: + self._parent[item] = self.find(self._parent[item]) + return self._parent[item] + + def union(self, left: int, right: int) -> None: + left_root = self.find(left) + right_root = self.find(right) + if left_root == right_root: + return + + if self._rank[left_root] < self._rank[right_root]: + self._parent[left_root] = right_root + elif self._rank[left_root] > self._rank[right_root]: + self._parent[right_root] = left_root + else: + self._parent[right_root] = left_root + self._rank[left_root] += 1 + + +def _build_vectorizer(config: TemplateGroupingConfig) -> TfidfVectorizer: + """예측이 아닌 그룹화 전용 문자 TF-IDF 벡터라이저 생성""" + return TfidfVectorizer( + analyzer="char_wb", + ngram_range=config.ngram_range, + min_df=config.min_df, + max_features=config.max_features, + lowercase=False, + dtype=np.float32, + norm="l2", + ) + + +def _find_similar_pairs( + texts: list[str], + config: TemplateGroupingConfig, +) -> list[tuple[int, int, float]]: + """임계값 이상의 cosine similarity를 가진 인덱스 쌍을 반환합니다.""" + if len(texts) < 2 or not any(text.strip() for text in texts): + return [] + + matrix = _build_vectorizer(config).fit_transform(texts) + radius = 1.0 - config.similarity_threshold + + neighbors = NearestNeighbors( + metric="cosine", + algorithm="brute", + radius=radius, + n_jobs=-1, + ).fit(matrix) + distances, indices = neighbors.radius_neighbors( + matrix, + return_distance=True, + sort_results=True, + ) + + pairs: list[tuple[int, int, float]] = [] + for left, (row_distances, row_indices) in enumerate( + zip(distances, indices, strict=True) + ): + for distance, right_value in zip( + row_distances, row_indices, strict=True + ): + right = int(right_value) + if left >= right: + continue + + similarity = 1.0 - float(distance) + if similarity >= config.similarity_threshold: + pairs.append((left, right, similarity)) + return pairs + + +def _create_stable_group_ids( + fingerprints: list[str], + union_find: UnionFind, +) -> list[str]: + """각 연결 요소의 최소 fingerprint로 안정적인 그룹 ID를 만듭니다.""" + members_by_root: dict[int, list[int]] = {} + for index in range(len(fingerprints)): + members_by_root.setdefault(union_find.find(index), []).append(index) + + ids_by_index: dict[int, str] = {} + for member_indices in members_by_root.values(): + representative = min(fingerprints[index] for index in member_indices) + group_id = f"tpl_{representative[:GROUP_ID_HASH_LENGTH]}" + for index in member_indices: + ids_by_index[index] = group_id + + return [ids_by_index[index] for index in range(len(fingerprints))] + + +def assign_template_groups( + df: pd.DataFrame, + *, + config: TemplateGroupingConfig | None = None, + text_column: str = "text_norm", +) -> pd.DataFrame: + """정규화 메시지에 유사도 기반 template_group_id를 부여합니다.""" + config = config or TemplateGroupingConfig() + if text_column not in df.columns: + raise ValueError(f"missing text column: {text_column}") + + result = df.copy().reset_index(drop=True) + if "text_fingerprint" not in result.columns: + result = add_text_fingerprints(result, text_column=text_column) + + if result.empty: + result["template_group_id"] = pd.Series(dtype="string") + return result + + texts = result[text_column].astype(str).tolist() + fingerprints = result["text_fingerprint"].astype(str).tolist() + union_find = UnionFind(len(result)) + + for left, right, _similarity in _find_similar_pairs(texts, config): + union_find.union(left, right) + + result["template_group_id"] = _create_stable_group_ids( + fingerprints, + union_find, + ) + return result diff --git a/data_science/SMSModel/tokenization/__init__.py b/data_science/SMSModel/tokenization/__init__.py new file mode 100644 index 0000000..f4717c6 --- /dev/null +++ b/data_science/SMSModel/tokenization/__init__.py @@ -0,0 +1,13 @@ +"""SMS 텍스트 모델에서 사용하는 tokenizer 패키지""" + +from .kiwi_tokenizer import ( + MASK_TOKENS, + SELECTED_POS_TAGS, + kiwi_tokenize, +) + +__all__ = [ + "MASK_TOKENS", + "SELECTED_POS_TAGS", + "kiwi_tokenize", +] \ No newline at end of file diff --git a/data_science/SMSModel/tokenization/kiwi_tokenizer.py b/data_science/SMSModel/tokenization/kiwi_tokenizer.py new file mode 100644 index 0000000..20fdad2 --- /dev/null +++ b/data_science/SMSModel/tokenization/kiwi_tokenizer.py @@ -0,0 +1,152 @@ +"""Kiwi를 이용한 한국어 SMS 형태소 tokenizer. + +이 모듈의 ``kiwi_tokenize`` 함수는 scikit-learn의 +CountVectorizer 또는 TfidfVectorizer에서 바로 사용할 수 있도록 +문자열을 입력받아 문자열 토큰 리스트를 반환합니다. +""" + +from __future__ import annotations + +import re +from functools import lru_cache + +from kiwipiepy import Kiwi + + +# 기존 공통 전처리에서 만들어지는 마스킹 토큰입니다. +# +# 형태소 분석기에 "[URL]" 전체를 전달하면 "[", "URL", "]"로 +# 분리될 수 있으므로 tokenizer에서 먼저 찾아 하나의 토큰으로 보존합니다. +MASK_TOKENS: frozenset[str] = frozenset( + { + "[URL]", + "[PHONE]", + "[ACCOUNT]", + "[AMOUNT]", + "[EMAIL]", + "[CARD]", + "[RRN]", + } +) + +# 정규식에서 긴 토큰이 짧은 토큰보다 먼저 매칭되도록 정렬합니다. +MASK_TOKEN_PATTERN = re.compile( + "(" + + "|".join( + re.escape(token) + for token in sorted( + MASK_TOKENS, + key=len, + reverse=True, + ) + ) + + ")" +) + + +# 분류에 사용할 품사를 명시적으로 정의합니다. +# +# NNG: 일반 명사 +# NNP: 고유 명사 +# NNB: 의존 명사 +# NR: 수사 +# NP: 대명사 +# VV: 동사 +# VA: 형용사 +# VX: 보조 용언 +# VCP: 긍정 지정사 +# VCN: 부정 지정사 +# MM: 관형사 +# MAG: 일반 부사 +# MAJ: 접속 부사 +# XR: 어근 +# SL: 영문/외국어 +# +# 조사(J*)와 어미(E*)는 메시지의 핵심 의미보다 문법적인 역할이 +# 크기 때문에 기본적으로 제외합니다. +SELECTED_POS_TAGS: frozenset[str] = frozenset( + { + "NNG", # 일반 명사 + "NNP", # 고유 명사 + "NNB", # 의존 명사 + "NR", # 수사 + "NP", # 대명사 + "VV", # 동사 + "VA", # 형용사 + "VX", # 보조 용언 + "VCP", # 긍정 지정사 + "VCN", # 부정 지정사 + "MM", # 관형사 + "MAG", # 일반 부사 + "MAJ", # 접속 부사 + "XR", # 어근 + "SL", # 영문/외국어 + } +) + + +@lru_cache(maxsize=1) +def _get_kiwi() -> Kiwi: + + """현재 Python 프로세스에서 Kiwi 인스턴스를 한 번만 생성""" + + return Kiwi() + + +def _base_pos_tag(tag: str) -> str: + + """Kiwi의 불규칙 활용 접미사를 제거한 기본 품사를 반환""" + + return tag.split("-", maxsplit=1)[0] + + +def _tokenize_segment(segment: str) -> list[str]: + """마스킹 토큰이 포함되지 않은 일반 문자열 조각 분석""" + if not segment or segment.isspace(): + return [] + + tokens: list[str] = [] + + # 초성체가 앞 음절의 받침으로 붙어 형태소 분석이 깨지는 경우 최소화 (ex: "했엌ㅋㅋ") + for token in _get_kiwi().tokenize( + segment, + normalize_coda=True, + ): + base_tag = _base_pos_tag(token.tag) + form = token.form.strip() + + if not form: + continue + + if base_tag not in SELECTED_POS_TAGS: + continue + + tokens.append(form) + + return tokens + + +def kiwi_tokenize(text: str) -> list[str]: + + """문자열을 분류용 형태소 토큰 리스트로 변환""" + + if not isinstance(text, str): + raise TypeError("text must be a string") + + if not text.strip(): + return [] + + result: list[str] = [] + + # re.split의 캡처 그룹으로 인해 마스킹 토큰도 결과에 포함 + for part in MASK_TOKEN_PATTERN.split(text): + if not part or part.isspace(): + continue + + if part in MASK_TOKENS: + result.append(part) + continue + + result.extend(_tokenize_segment(part)) + + return result \ No newline at end of file diff --git a/data_science/SMSModel/train_sms.py b/data_science/SMSModel/train_sms.py index fb23404..25145a7 100644 --- a/data_science/SMSModel/train_sms.py +++ b/data_science/SMSModel/train_sms.py @@ -1,21 +1,6 @@ -""" -SafeFam — SMS 피싱 탐지 베이즈 분류기 학습 파이프라인 v2 -train_sms.py -========================================================== -v2 변경사항: - - CalibratedClassifierCV(method='isotonic') 추가 - → ComplementNB 확률이 0/1 양극단으로 몰리는 문제 해결 - → risk_score가 0~100 구간에 고르게 분산되어 상/중/하 의미있는 구분 가능 - -탐지 파이프라인 내 위치: - URL 있음 → 베이즈(텍스트) + VirusTotal(URL) 병렬 → 종합 risk_score (Spring Boot 조율) - URL 없음 → 베이즈 단독 risk_score - MEDIUM → Claude API 에스컬레이션 (40~69점 구간) -""" - +# SMS 피싱 탐지 베이즈 분류기 학습 파이프라인 from __future__ import annotations -import re import warnings from pathlib import Path @@ -23,22 +8,66 @@ import numpy as np import pandas as pd from scipy.sparse import csr_matrix, hstack -from sklearn.calibration import CalibratedClassifierCV # ← v2 추가 +from sklearn.calibration import CalibratedClassifierCV from sklearn.feature_extraction.text import CountVectorizer from sklearn.metrics import classification_report, confusion_matrix -from sklearn.model_selection import StratifiedShuffleSplit from sklearn.naive_bayes import ComplementNB +from app.analysis.text.preprocessing import ( + extract_struct_feature_matrix, + normalize_text, +) +from data_science.SMSModel.template_grouping import ( + TemplateGroupingConfig, + prepare_template_groups, +) +from data_science.SMSModel.dataset_splitting import ( + DatasetSplitConfig, + DatasetSplits, + load_split_manifest, + save_split_manifest, + split_grouped_dataset, + validate_dataset_splits, +) +from data_science.SMSModel.reporting import ( + generate_dataset_split_reports, +) + warnings.filterwarnings("ignore") # ───────────────────────────────────────────────────────────────────────────── -# 경로 CONFIG — 환경에 따라 이 블록만 수정 +# 경로 CONFIG — 실행 위치가 아니라 이 파일의 위치를 기준으로 계산 # ───────────────────────────────────────────────────────────────────────────── -DATA_PATH = Path("../Data/SMSData/phishing_total_dataset_2705.csv") -MODEL_PATH = Path("phishing_model_artifact.pkl") -VECTORIZER_PATH = Path("phishing_vectorizer.pkl") +SMS_MODEL_DIR = Path(__file__).resolve().parent +DATA_PATH = ( + SMS_MODEL_DIR.parent + / "Data" + / "SMSData" + / "phishing_total_dataset_2705.csv" +) +ARTIFACTS_DIR = SMS_MODEL_DIR / "artifacts" +MODEL_PATH = ARTIFACTS_DIR / "phishing_model_artifact.pkl" +VECTORIZER_PATH = ARTIFACTS_DIR / "phishing_vectorizer.pkl" +SPLIT_MANIFEST_PATH = ( + SMS_MODEL_DIR + / "splits" + / "sms_split_v1.csv" +) + +# 보고서 경로 +REPORTS_DIR = SMS_MODEL_DIR / "reports" + +DATASET_SPLIT_JSON_REPORT_PATH = ( + REPORTS_DIR + / "dataset_split_summary.json" +) + +DATASET_SPLIT_MARKDOWN_REPORT_PATH = ( + REPORTS_DIR + / "dataset_split_summary.md" +) # ───────────────────────────────────────────────────────────────────────────── # 학습 CONFIG @@ -58,107 +87,143 @@ ALPHA_GRID = [0.01, 0.05, 0.1, 0.3, 0.5, 1.0, 2.0, 5.0] THRESHOLD_GRID = np.round(np.arange(0.30, 0.75, 0.05), 2) - # ───────────────────────────────────────────────────────────────────────────── -# 정규화 패턴 -# Spring PiiMaskingService 토큰과 일치: [PHONE],[ACCOUNT],[CARD],[RRN],[EMAIL] -# 마스킹 순서: RRN→CARD→PHONE→ACCOUNT→EMAIL (Spring과 동일하게 유지) +# 유사 템플릿 그룹화 CONFIG # ───────────────────────────────────────────────────────────────────────────── -_RE_URL = re.compile(r"(?i)(? str: - parts = [] - last_end = 0 - for m in _RE_URL.finditer(text): - parts.append(_mask_pii(text[last_end:m.start()])) - parts.append("[URL]") - last_end = m.end() - parts.append(_mask_pii(text[last_end:])) - text = "".join(parts) - text = _RE_AMOUNT.sub("[AMOUNT]", text) - text = _RE_FORMAT_ARTIFACT.sub(" ", text) - return re.sub(r"\s+", " ", text).strip() - -def _mask_pii(text: str) -> str: - text = _RE_RRN.sub("[RRN]", text) - text = _RE_CARD.sub("[CARD]", text) - text = _RE_PHONE.sub("[PHONE]", text) - text = _RE_ACCOUNT.sub("[ACCOUNT]", text) - text = _RE_EMAIL.sub("[EMAIL]", text) - return text - - -def _extract_struct_features(texts: pd.Series, has_url: pd.Series) -> np.ndarray: - """ - 텍스트 n-gram으로 포착하기 어려운 구조적 신호 6개를 boolean 피처로 추출. - 반환 shape: (n_samples, 6) - """ - return np.column_stack([ - has_url.astype(int).values, # 0: URL 포함 - texts.str.contains(_RE_SHORT_URL).astype(int).values, # 1: 단축URL (강신호) - (texts.str.contains(_RE_PHONE) | texts.str.contains(r"\[PHONE\]", regex=True)).astype(int).values, # 2: 전화번호 - texts.str.contains(_RE_AMOUNT).astype(int).values, # 3: 금액 - texts.str.contains(_RE_WEB_TAG).astype(int).values, # 4: 통신사태그(역방향) - (texts.str.len() > 100).astype(int).values, # 5: 100자 초과 - ]) +def build_template_grouping_config() -> TemplateGroupingConfig: + """현재 학습 실행에서 사용할 템플릿 그룹화 설정을 반환""" + return TemplateGroupingConfig( + similarity_threshold=TEMPLATE_SIMILARITY_THRESHOLD, + ngram_range=TEMPLATE_NGRAM_RANGE, + min_df=1, + max_features=TEMPLATE_MAX_FEATURES, + ) +def build_dataset_split_config() -> DatasetSplitConfig: + """현재 SMS 모델 학습에서 사용할 데이터 분할 설정""" + return DatasetSplitConfig( + train_size=0.70, + val_size=0.15, + test_size=0.15, + random_state=42, + candidate_count=500, + ) # ───────────────────────────────────────────────────────────────────────────── # 데이터 로드 # ───────────────────────────────────────────────────────────────────────────── def load_data(path: Path) -> tuple[pd.DataFrame, pd.DataFrame]: - """ - CSV 로드 → 무결성 검증 → 정규화 → text_norm 기준 중복 제거. + """CSV를 읽고 학습용 데이터와 별도 holdout 데이터를 반환""" + df = pd.read_csv(path) - source == 'synthetic_new_holdout' / 'synthetic_fp_stress' 행은 학습/분리 - 대상에서 제외하고 df_holdout으로 따로 반환 (voice 모델과 동일 패턴). + required_columns = { + "text", + "label", + "type", + "has_url", + } + missing_columns = required_columns - set(df.columns) - Returns: - (df, df_holdout) - """ - df = pd.read_csv(path) + if missing_columns: + raise ValueError(f"누락 컬럼: {missing_columns}") - required_cols = {"text", "label", "type", "has_url"} - if missing := required_cols - set(df.columns): - raise ValueError(f"누락 컬럼: {missing}") if df.isnull().any().any(): raise ValueError("결측치가 존재합니다.") - if not {"phishing", "normal"}.issuperset(set(df["label"].unique())): - raise ValueError(f"예상치 못한 label 값: {df['label'].unique()}") - print(f"[Load] 원본 {len(df)}건 | {df['label'].value_counts().to_dict()}") + allowed_labels = {"phishing", "normal"} + actual_labels = set(df["label"].unique()) + + if not allowed_labels.issuperset(actual_labels): + raise ValueError( + f"예상치 못한 label 값: {df['label'].unique()}" + ) + + print( + f"[Load] 원본 {len(df)}건 | " + f"{df['label'].value_counts().to_dict()}" + ) + + # 학습과 API가 공유하는 공통 정규화 함수를 사용 + df["text_norm"] = df["text"].apply(normalize_text) + + source = df.get( + "source", + pd.Series("original", index=df.index), + ) + + is_new_holdout = source.isin( + [ + "synthetic_new_holdout", + "synthetic_fp_stress", + ] + ) + + # 신규 시나리오 holdout은 학습 데이터 그룹화 대상에서도 제외 + df_holdout = ( + df[is_new_holdout] + .copy() + .reset_index(drop=True) + ) + + df_pool = ( + df[~is_new_holdout] + .copy() + .reset_index(drop=True) + ) + + before_deduplication = len(df_pool) + + # fingerprint 생성 → label 충돌 검사 → 완전 중복 제거 → 유사 그룹화 + df_pool = prepare_template_groups( + df_pool, + config=build_template_grouping_config(), + text_column="text_norm", + label_column="label", + ) - df["text_norm"] = df["text"].apply(_normalize_text) + removed_duplicates = before_deduplication - len(df_pool) + template_group_count = df_pool["template_group_id"].nunique() - is_new_holdout = df.get("source", pd.Series("original", index=df.index)).isin( - ["synthetic_new_holdout", "synthetic_fp_stress"] + group_sizes = df_pool["template_group_id"].value_counts() + similar_group_count = int((group_sizes > 1).sum()) + largest_group_size = ( + int(group_sizes.max()) + if not group_sizes.empty + else 0 ) - df_holdout = df[is_new_holdout].reset_index(drop=True) - df_pool = df[~is_new_holdout].reset_index(drop=True) - before = len(df_pool) - df_pool = df_pool.drop_duplicates(subset="text_norm").reset_index(drop=True) print( - f"[Dedup] text_norm 기준 중복 제거: {before} → {len(df_pool)}건 | " + f"[Dedup] fingerprint 기준 완전 중복 제거: " + f"{before_deduplication} → {len(df_pool)}건 " + f"(제거 {removed_duplicates}건)" + ) + + print( + f"[Template Grouping] 전체 그룹={template_group_count} | " + f"유사 메시지 그룹={similar_group_count} | " + f"최대 그룹 크기={largest_group_size}" + ) + + print( + f"[Pool] label 분포: " f"{df_pool['label'].value_counts().to_dict()}" ) - print(f"[Holdout] 완전 신규 시나리오 {len(df_holdout)}건 분리 (학습에 전혀 사용 안 됨)") + + print( + f"[Holdout] 완전 신규 시나리오 {len(df_holdout)}건 분리 " + f"(학습에 전혀 사용 안 됨)" + ) return df_pool, df_holdout @@ -167,42 +232,96 @@ def load_data(path: Path) -> tuple[pd.DataFrame, pd.DataFrame]: # Train / Test 분리 # ───────────────────────────────────────────────────────────────────────────── -def split_data(df: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: - """ - StratifiedShuffleSplit 2단계: label 비율 유지하며 train/val/test 분리. +def split_data( + df: pd.DataFrame, + *, + create_manifest: bool = False, +) -> DatasetSplits: + """저장된 manifest를 사용하거나 새로운 그룹 split을 생성""" + split_config = build_dataset_split_config() + grouping_config = build_template_grouping_config() + + if SPLIT_MANIFEST_PATH.exists() and not create_manifest: + splits = load_split_manifest( + df, + SPLIT_MANIFEST_PATH, + config=split_config, + ) - 1단계) 전체 → (train+val) / test - 2단계) (train+val) → train / val (val 비율을 전체 기준 VAL_SIZE로 재조정) + print( + f"[Split] 기존 manifest 사용: " + f"{SPLIT_MANIFEST_PATH}" + ) + else: + splits = split_grouped_dataset( + df, + config=split_config, + ) - val: 하이퍼파라미터(alpha/threshold) 튜닝 전용 - test: 최종 평가 1회 전용 — 튜닝에 재사용하면 지표가 낙관적으로 부풀려짐(test set 누수) - """ - sss_test = StratifiedShuffleSplit( - n_splits=1, test_size=TEST_SIZE, random_state=RANDOM_STATE - ) - trainval_idx, test_idx = next(sss_test.split(df, df["label"])) - df_trainval = df.iloc[trainval_idx].reset_index(drop=True) - df_test = df.iloc[test_idx].reset_index(drop=True) + validate_dataset_splits( + df, + splits, + config=split_config, + ) - val_ratio = VAL_SIZE / (1 - TEST_SIZE) # trainval 내 비중으로 재조정 - sss_val = StratifiedShuffleSplit( - n_splits=1, test_size=val_ratio, random_state=RANDOM_STATE + save_split_manifest( + splits, + SPLIT_MANIFEST_PATH, + # create_manifest가 명시된 경우만 기존 파일 변경을 허용합니다. + overwrite=create_manifest, + ) + + print( + f"[Split] 새 manifest 저장: " + f"{SPLIT_MANIFEST_PATH}" + ) + + # manifest를 로드한 경우에도 학습 직전에 다시 검증합니다. 실패 시 예외가 + # 전파되어 모델 학습과 잘못된 보고서 생성을 모두 중단합니다. + validate_dataset_splits(df, splits, config=split_config) + + summary = generate_dataset_split_reports( + df, + splits, + split_config=split_config, + grouping_config=grouping_config, + json_path=DATASET_SPLIT_JSON_REPORT_PATH, + markdown_path=DATASET_SPLIT_MARKDOWN_REPORT_PATH, ) - train_idx, val_idx = next(sss_val.split(df_trainval, df_trainval["label"])) - df_train = df_trainval.iloc[train_idx].reset_index(drop=True) - df_val = df_trainval.iloc[val_idx].reset_index(drop=True) - - def _fmt(d: pd.DataFrame) -> str: - vc = d["label"].value_counts() - return ( - f"phishing={vc.get('phishing', 0)} ({vc.get('phishing', 0)/len(d):.1%}) | " - f"normal={vc.get('normal', 0)} ({vc.get('normal', 0)/len(d):.1%})" + + def describe_split( + name: str, + split_df: pd.DataFrame, + ) -> None: + label_counts = split_df["label"].value_counts() + group_count = split_df["template_group_id"].nunique() + + print( + f"[Split] {name:<10} " + f"rows={len(split_df)} | " + f"groups={group_count} | " + f"phishing={label_counts.get('phishing', 0)} " + f"({(split_df['label'] == 'phishing').mean():.1%}) | " + f"normal={label_counts.get('normal', 0)} " + f"({(split_df['label'] == 'normal').mean():.1%})" ) - print(f"[Split] Train {len(df_train)}건: {_fmt(df_train)}") - print(f"[Split] Val {len(df_val)}건: {_fmt(df_val)}") - print(f"[Split] Test {len(df_test)}건: {_fmt(df_test)}") - return df_train, df_val, df_test + describe_split("Train", splits.train) + describe_split("Validation", splits.validation) + describe_split("Test", splits.test) + + print( + "[Split Validation] " + f"passed={summary['validation']['passed']} | " + f"group_overlap={summary['validation']['group_overlap_count']} | " + "fingerprint_overlap=" + f"{summary['validation']['fingerprint_overlap_count']}" + ) + print(f"[Dataset] fingerprint={summary['dataset_fingerprint']}") + print(f"[Report] {DATASET_SPLIT_JSON_REPORT_PATH}") + print(f"[Report] {DATASET_SPLIT_MARKDOWN_REPORT_PATH}") + + return splits # ───────────────────────────────────────────────────────────────────────────── @@ -373,6 +492,7 @@ def evaluate(model, threshold: float, X_test, y_test: pd.Series) -> None: def save_artifacts(model, vectorizer: CountVectorizer, threshold: float) -> None: """model + threshold + classes를 단일 아티팩트로 저장 (FastAPI 로드용).""" + ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True) joblib.dump( {"model": model, "threshold": threshold, "classes": list(model.classes_)}, MODEL_PATH, @@ -417,60 +537,46 @@ def predict_risk_score( classes: list, ) -> dict: """ - 단일 문자 텍스트 → risk_score(0~100), risk_level(HIGH/MEDIUM/LOW) 반환. - - v2: CalibratedClassifierCV 보정으로 prob_phishing이 0~1 고르게 분산됨 - → risk_score가 실제 위험도를 반영하는 연속적 점수로 의미있게 동작 - - Args: - text : 원문 SMS 텍스트 - model : CalibratedClassifierCV (ComplementNB + isotonic 보정) - vectorizer: 학습된 CountVectorizer - threshold : 최적화된 분류 임계값 - classes : model.classes_ 리스트 - - Returns: - { - "risk_score" : int (0~100, 보정된 피싱 확률 × 100) - "risk_level" : str ("HIGH" | "MEDIUM" | "LOW") - "text_score_only": bool (True = VirusTotal 결합 전 텍스트 단독 점수) - } + 단일 SMS를 분석해 위험 점수와 위험 등급을 반환 + + 학습과 운영 API 모두 공통 normalize_text()와 extract_struct_feature_matrix()를 사용 """ - text_norm = _normalize_text(text) - - # 단일 샘플 구조적 피처 (shape: 1 × 6) - struct = np.array([[ - int(bool(_RE_URL.search(text))), - int(bool(_RE_SHORT_URL.search(text))), - int(bool(_RE_PHONE.search(text) or "[PHONE]" in text)), - int(bool(_RE_AMOUNT.search(text))), - int(bool(_RE_WEB_TAG.search(text))), - int(len(text) > 100), - ]]) - - X = build_feature_matrix(vectorizer, pd.Series([text_norm]), struct, fit=False) - prob_phish = model.predict_proba(X)[0][classes.index("phishing")] - risk_score = int(prob_phish * 100) + text_norm = normalize_text(text) + + struct = extract_struct_feature_matrix([text]) + + X = build_feature_matrix( + vectorizer, + pd.Series([text_norm]), + struct, + fit=False, + ) + + phishing_index = classes.index("phishing") + phishing_probability = model.predict_proba(X)[0][phishing_index] + risk_score = int(phishing_probability * 100) return { - "risk_score" : risk_score, - "risk_level" : _map_risk_level(risk_score), - "text_score_only": True, # VirusTotal 결합 전 텍스트 단독 점수 + "risk_score": risk_score, + "risk_level": _map_risk_level(risk_score), + "text_score_only": True, } - # ───────────────────────────────────────────────────────────────────────────── # 진입점 # ───────────────────────────────────────────────────────────────────────────── def main() -> None: - df, df_holdout = load_data(DATA_PATH) # 반환값 2개로 변경 + df, df_holdout = load_data(DATA_PATH) - df_train, df_val, df_test = split_data(df) + splits = split_data(df) + df_train = splits.train + df_val = splits.validation + df_test = splits.test - struct_train = _extract_struct_features(df_train["text"], df_train["has_url"]) - struct_val = _extract_struct_features(df_val["text"], df_val["has_url"]) - struct_test = _extract_struct_features(df_test["text"], df_test["has_url"]) + struct_train = extract_struct_feature_matrix(df_train["text"], df_train["has_url"]) + struct_val = extract_struct_feature_matrix(df_val["text"], df_val["has_url"]) + struct_test = extract_struct_feature_matrix(df_test["text"], df_test["has_url"]) vectorizer = build_vectorizer() X_train = build_feature_matrix(vectorizer, df_train["text_norm"], struct_train, fit=True) @@ -491,7 +597,6 @@ def main() -> None: verify_probability_distribution(best["model"], X_test, df_test["label"]) evaluate(best["model"], best["threshold"], X_test, df_test["label"]) - # ★ 추가: 완전 신규 시나리오 일반화 + 오탐 검증 evaluate_new_holdout(best["model"], vectorizer, best["threshold"], df_holdout) save_artifacts(best["model"], vectorizer, best["threshold"]) @@ -506,8 +611,8 @@ def evaluate_new_holdout(model, vectorizer: CountVectorizer, threshold: float, print("\n[SKIP] 신규 holdout 데이터 없음") return - text_norm = df_holdout["text"].apply(_normalize_text) - struct = _extract_struct_features(df_holdout["text"], df_holdout["has_url"]) + text_norm = df_holdout["text"].apply(normalize_text) + struct = extract_struct_feature_matrix(df_holdout["text"], df_holdout["has_url"]) X = build_feature_matrix(vectorizer, text_norm, struct, fit=False) y_prob = model.predict_proba(X)[:, _phishing_idx(model)] @@ -542,4 +647,4 @@ def evaluate_new_holdout(model, vectorizer: CountVectorizer, threshold: float, lambda g: (g["pred"] == "phishing").mean() ).sort_values(ascending=False)) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/pytest.ini b/pytest.ini index 70361c0..9cb9d08 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ [pytest] markers = integration: hits real external hosts over the network; excluded by default, run explicitly with -m integration + slow: performs relatively expensive local data processing addopts = -m "not integration" asyncio_mode = auto diff --git a/requirements.txt b/requirements.txt index c512a53..8a99427 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,7 @@ python-dotenv>=1.0.0 scikit-learn==1.8.0 scipy>=1.16.0 numpy>=2.0.0,<2.5.0 +pandas>=2.2.0,<3.0.0 joblib>=1.5.0 aio-pika>=9.4.0,<10.0.0 +kiwipiepy==0.23.2 diff --git a/tests/analysis/text/test_naive_bayes_analyzer.py b/tests/analysis/text/test_naive_bayes_analyzer.py index 779f670..2ed7bfd 100644 --- a/tests/analysis/text/test_naive_bayes_analyzer.py +++ b/tests/analysis/text/test_naive_bayes_analyzer.py @@ -1,18 +1,24 @@ -import pytest +"""Naive Bayes API 분석기의 모델 로딩 및 응답 회귀 테스트.""" + from pathlib import Path + +import pytest + from app.analysis.text import naive_bayes_analyzer as nb @pytest.fixture(autouse=True) -def _reset_model_cache(): - """모듈 전역 캐시가 테스트 간에 새서 로드 성공/실패 케이스가 서로 오염되지 않도록 초기화.""" +def reset_model_cache(): + """모델 전역 캐시가 테스트 사이에 공유되지 않도록 초기화합니다.""" nb._model = None nb._vectorizer = None nb._threshold = None nb._classes = None nb._load_error = None nb._load_attempted = False + yield + nb._model = None nb._vectorizer = None nb._threshold = None @@ -21,64 +27,52 @@ def _reset_model_cache(): nb._load_attempted = False -def test_normalize_text_masks_url_and_amount(): - text = "http://bit.ly/fake 계좌로 500,000원 즉시 입금하세요" - normalized = nb._normalize_text(text) - - assert "[URL]" in normalized - assert "[AMOUNT]" in normalized - assert "http://" not in normalized - - -def test_extract_struct_features_detects_short_url_and_phone(): - text = "010-1234-5678 로 연락주세요 http://bit.ly/fake" - features = nb._extract_struct_features(text) - - # [has_url, has_short_url, has_phone, has_amount, has_web_tag, is_long_text] - assert features[0] == 1 # has_url - assert features[1] == 1 # has_short_url - assert features[2] == 1 # has_phone - assert features[3] == 0 # has_amount - assert features[4] == 0 # has_web_tag - assert features[5] == 0 # is_long_text - - @pytest.mark.asyncio -async def test_analyze_text_with_naive_bayes_missing_artifact_is_fail_safe(monkeypatch): - """ - 모델 아티팩트를 찾을 수 없을 때 예외를 던지는 대신 UNKNOWN 등급 + 에러 메시지로 - 안전하게 대체되는지 검증합니다 (fail-safe fallback). - """ - monkeypatch.setattr(nb, "MODEL_PATH", Path("/nonexistent/model.pkl")) - monkeypatch.setattr(nb, "VECTORIZER_PATH", Path("/nonexistent/vectorizer.pkl")) +async def test_missing_artifact_returns_fail_safe_result( + monkeypatch, + tmp_path: Path, +): + """아티팩트가 없을 때 예외 대신 UNKNOWN 결과를 반환합니다.""" + monkeypatch.setattr(nb, "MODEL_PATH", tmp_path / "missing-model.pkl") + monkeypatch.setattr( + nb, + "VECTORIZER_PATH", + tmp_path / "missing-vectorizer.pkl", + ) result = await nb.analyze_text_with_naive_bayes("테스트 메시지") + assert result["engine"] == "naive_bayes" assert result["is_available"] is False assert result["result"]["grade"] == "UNKNOWN" + assert result["result"]["risk_score"] == 0 + assert result["result"]["is_suspected_phishing"] is False assert result["result"]["error_message"] is not None @pytest.mark.asyncio -async def test_analyze_text_with_naive_bayes_real_model_flags_phishing(): - """ - 저장소에 실제로 커밋된 사전 학습 아티팩트를 로드하여, 전형적인 스미싱 문장이 - 높은 위험 점수로 판정되는지 통합 검증합니다. - """ - text = "[국민건강보험] 건강검진 보고서 발급 완료. 즉시 확인하세요 http://bit.ly/fake" +async def test_real_model_flags_phishing(): + """기존 모델이 전형적인 피싱 문장을 계속 탐지하는지 확인합니다.""" + text = ( + "[국민건강보험] 건강검진 보고서 발급 완료. " + "즉시 확인하세요 http://bit.ly/fake" + ) result = await nb.analyze_text_with_naive_bayes(text) assert result["is_available"] is True assert result["result"]["risk_score"] > 50 assert result["result"]["grade"] in ("SUSPICIOUS", "DANGEROUS") + assert result["result"]["is_suspected_phishing"] is True + assert result["result"]["error_message"] is None @pytest.mark.asyncio -async def test_analyze_text_with_naive_bayes_real_model_flags_casual_text_as_safe(): - text = "오늘 소주 한잔 고?" - - result = await nb.analyze_text_with_naive_bayes(text) +async def test_real_model_flags_casual_text_as_safe(): + """기존 모델이 일반 대화를 계속 안전으로 판단하는지 확인합니다.""" + result = await nb.analyze_text_with_naive_bayes("오늘 소주 한잔 고?") assert result["is_available"] is True assert result["result"]["grade"] == "SAFE" + assert result["result"]["is_suspected_phishing"] is False + assert result["result"]["error_message"] is None diff --git a/tests/analysis/text/test_preprocessing.py b/tests/analysis/text/test_preprocessing.py new file mode 100644 index 0000000..ea7f53e --- /dev/null +++ b/tests/analysis/text/test_preprocessing.py @@ -0,0 +1,164 @@ +import numpy as np +import pytest + +from app.analysis.text.preprocessing import ( + STRUCT_FEATURE_NAMES, + extract_struct_feature_matrix, + extract_struct_features, + mask_pii, + normalize_text, +) + + +@pytest.mark.parametrize( + ("original", "expected"), + [ + ( + "010-1234-5678로 연락주세요", + "[PHONE]로 연락주세요", + ), + ( + "test@example.com으로 보내주세요", + "[EMAIL]으로 보내주세요", + ), + ( + "카드번호 1234-5678-9012-3456", + "카드번호 [CARD]", + ), + ( + "주민번호 900101-1234567", + "주민번호 [RRN]", + ), + ( + "계좌 123-456-789012로 입금", + "계좌 [ACCOUNT]로 입금", + ), + ], +) +def test_mask_pii_replaces_personal_information( + original: str, + expected: str, +): + assert mask_pii(original) == expected + + +def test_normalize_text_masks_url_and_amount(): + text = ( + "http://bit.ly/fake 계좌로 " + "500,000원 즉시 입금하세요" + ) + + normalized = normalize_text(text) + + assert normalized == ( + "[URL] 계좌로 [AMOUNT] 즉시 입금하세요" + ) + + +def test_normalize_text_collapses_whitespace(): + text = " 긴급\t\t확인\n\n필요 " + + normalized = normalize_text(text) + + assert normalized == "긴급 확인 필요" + + +def test_normalize_text_removes_format_artifacts(): + text = "■■ 긴급 == 확인 □□" + + normalized = normalize_text(text) + + assert normalized == "긴급 확인" + + +def test_normalize_text_does_not_mask_digits_inside_url(): + text = "확인 주소 https://example.com/01012345678" + + normalized = normalize_text(text) + + assert normalized == "확인 주소 [URL]" + assert "[PHONE]" not in normalized + + +def test_normalize_text_rejects_non_string_input(): + with pytest.raises(TypeError, match="text must be a string"): + normalize_text(None) # type: ignore[arg-type] + + +def test_extract_struct_features_detects_expected_signals(): + text = ( + "[Web발신] 010-1234-5678로 연락하세요. " + "500,000원 확인: http://bit.ly/fake" + ) + + features = extract_struct_features(text) + + assert features == [ + 1, # has_url + 1, # has_short_url + 1, # has_phone + 1, # has_amount + 1, # has_web_tag + 0, # is_long_text + ] + + +def test_extract_struct_features_detects_long_text(): + text = "가" * 101 + + features = extract_struct_features(text) + + assert features[-1] == 1 + + +def test_extract_struct_features_accepts_precomputed_url_flag(): + features = extract_struct_features( + "URL이 제거된 메시지", + has_url=True, + ) + + assert features[0] == 1 + + +def test_extract_struct_feature_matrix_returns_expected_shape(): + texts = [ + "일반 메시지", + "http://bit.ly/fake 확인", + ] + + matrix = extract_struct_feature_matrix(texts) + + assert matrix.shape == (2, len(STRUCT_FEATURE_NAMES)) + assert matrix.dtype == np.int8 + assert matrix[0, 0] == 0 + assert matrix[1, 0] == 1 + assert matrix[1, 1] == 1 + + +def test_extract_struct_feature_matrix_uses_precomputed_url_flags(): + texts = [ + "URL이 제거된 첫 번째 메시지", + "URL이 없는 두 번째 메시지", + ] + has_urls = [True, False] + + matrix = extract_struct_feature_matrix(texts, has_urls) + + assert matrix[:, 0].tolist() == [1, 0] + + +def test_extract_struct_feature_matrix_rejects_length_mismatch(): + with pytest.raises( + ValueError, + match="texts and has_urls must contain the same number", + ): + extract_struct_feature_matrix( + ["첫 번째", "두 번째"], + [True], + ) + + +def test_extract_struct_feature_matrix_handles_empty_input(): + matrix = extract_struct_feature_matrix([]) + + assert matrix.shape == (0, len(STRUCT_FEATURE_NAMES)) \ No newline at end of file diff --git a/tests/data_science/SMSModel/__init__.py b/tests/data_science/SMSModel/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/data_science/SMSModel/evaluation/__init__.py b/tests/data_science/SMSModel/evaluation/__init__.py new file mode 100644 index 0000000..e5554a2 --- /dev/null +++ b/tests/data_science/SMSModel/evaluation/__init__.py @@ -0,0 +1 @@ +"""공통 모델 평가기 테스트 패키지.""" diff --git a/tests/data_science/SMSModel/evaluation/conftest.py b/tests/data_science/SMSModel/evaluation/conftest.py new file mode 100644 index 0000000..e63291a --- /dev/null +++ b/tests/data_science/SMSModel/evaluation/conftest.py @@ -0,0 +1,77 @@ +"""공통 평가기 테스트에서 사용하는 가짜 모델과 데이터.""" + +import numpy as np +import pandas as pd +import pytest + +from data_science.SMSModel.modeling import ( + BasePhishingClassifier, + ScoreOutput, + ScoreType, +) + + +class FakeProbabilityClassifier(BasePhishingClassifier): + """DataFrame의 mock_score를 그대로 반환하는 테스트 전용 모델.""" + + def __init__(self) -> None: + self.fitted = False + self.fit_input: pd.DataFrame | None = None + self.predict_call_count = 0 + + @property + def model_name(self) -> str: + return "fake_probability" + + @property + def score_type(self) -> ScoreType: + return ScoreType.PROBABILITY + + def fit(self, train_df: pd.DataFrame) -> BasePhishingClassifier: + self.fitted = True + self.fit_input = train_df.copy() + return self + + def predict_scores(self, df: pd.DataFrame) -> ScoreOutput: + if not self.fitted: + raise RuntimeError("model is not fitted") + + self.predict_call_count += 1 + return ScoreOutput( + values=df["mock_score"].to_numpy(dtype=float), + score_type=self.score_type, + ) + + def get_metadata(self) -> dict[str, object]: + return { + **super().get_metadata(), + "test_adapter": True, + } + + +@pytest.fixture +def fake_model() -> FakeProbabilityClassifier: + return FakeProbabilityClassifier() + + +@pytest.fixture +def evaluation_frames() -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + train = pd.DataFrame( + { + "label": ["normal", "phishing", "normal", "phishing"], + "mock_score": [0.1, 0.9, 0.2, 0.8], + } + ) + validation = pd.DataFrame( + { + "label": ["normal", "normal", "phishing", "phishing"], + "mock_score": [0.1, 0.4, 0.6, 0.9], + } + ) + test = pd.DataFrame( + { + "label": ["normal", "normal", "phishing", "phishing"], + "mock_score": [0.2, 0.5, 0.7, 0.95], + } + ) + return train, validation, test diff --git a/tests/data_science/SMSModel/evaluation/test_base.py b/tests/data_science/SMSModel/evaluation/test_base.py new file mode 100644 index 0000000..4155c11 --- /dev/null +++ b/tests/data_science/SMSModel/evaluation/test_base.py @@ -0,0 +1,73 @@ +"""공통 모델 인터페이스와 score 계약 테스트.""" + +import numpy as np +import pandas as pd +import pytest + +from data_science.SMSModel.modeling import ScoreOutput, ScoreType + + +def test_probability_scores_accept_zero_and_one(): + output = ScoreOutput( + values=np.asarray([0.0, 0.5, 1.0]), + score_type=ScoreType.PROBABILITY, + ) + assert output.values.tolist() == [0.0, 0.5, 1.0] + + +@pytest.mark.parametrize("invalid_value", [-0.01, 1.01]) +def test_probability_scores_reject_out_of_range_values(invalid_value): + with pytest.raises(ValueError, match="between 0 and 1"): + ScoreOutput( + values=np.asarray([invalid_value]), + score_type=ScoreType.PROBABILITY, + ) + + +def test_decision_scores_allow_unbounded_values(): + output = ScoreOutput( + values=np.asarray([-3.5, 0.0, 7.2]), + score_type=ScoreType.DECISION, + ) + assert output.values.tolist() == [-3.5, 0.0, 7.2] + + +@pytest.mark.parametrize( + "values, message", + [ + (np.asarray([[0.1, 0.2]]), "one-dimensional"), + (np.asarray([0.1, np.nan]), "finite"), + (np.asarray([0.1, np.inf]), "finite"), + ], +) +def test_score_output_rejects_invalid_shape_or_values(values, message): + with pytest.raises(ValueError, match=message): + ScoreOutput(values=values, score_type=ScoreType.PROBABILITY) + + +def test_predict_uses_probability_default_threshold(fake_model): + fake_model.fit(pd.DataFrame({"label": ["normal"], "mock_score": [0.1]})) + frame = pd.DataFrame({"mock_score": [0.49, 0.5, 0.51]}) + + assert fake_model.default_threshold == 0.5 + assert fake_model.predict(frame).tolist() == [ + "normal", + "phishing", + "phishing", + ] + + +def test_predict_accepts_custom_threshold_and_metadata(fake_model): + fake_model.fit(pd.DataFrame({"label": ["normal"], "mock_score": [0.1]})) + frame = pd.DataFrame({"mock_score": [0.6, 0.8]}) + + assert fake_model.predict(frame, threshold=0.7).tolist() == [ + "normal", + "phishing", + ] + assert fake_model.get_metadata() == { + "model_name": "fake_probability", + "score_type": "probability", + "default_threshold": 0.5, + "test_adapter": True, + } diff --git a/tests/data_science/SMSModel/evaluation/test_evaluator.py b/tests/data_science/SMSModel/evaluation/test_evaluator.py new file mode 100644 index 0000000..1e9317b --- /dev/null +++ b/tests/data_science/SMSModel/evaluation/test_evaluator.py @@ -0,0 +1,65 @@ +"""공통 학습·validation 선택·test 평가 흐름 테스트.""" + +import pytest + +from data_science.SMSModel.evaluation.evaluator import ( + train_and_evaluate_model, +) + + +def test_trains_selects_validation_threshold_and_evaluates_test( + fake_model, + evaluation_frames, +): + train, validation, test = evaluation_frames + + result = train_and_evaluate_model( + fake_model, + train_df=train, + validation_df=validation, + test_df=test, + target_recall=1.0, + latency_sample_count=2, + ) + + assert fake_model.fitted is True + assert fake_model.fit_input.equals(train) + assert result.model_name == "fake_probability" + assert result.score_type == "probability" + assert result.selected_threshold == pytest.approx(0.6) + assert result.validation.recall == pytest.approx(1.0) + assert result.test_metrics.true_negative == 2 + assert result.test_metrics.true_positive == 2 + assert result.test_metrics.false_negative == 0 + assert result.metadata["test_adapter"] is True + assert result.latency.sample_count == 2 + assert result.to_dict()["test_metrics"]["recall"] == pytest.approx(1.0) + + +def test_test_labels_do_not_change_selected_validation_threshold( + fake_model, + evaluation_frames, +): + train, validation, test = evaluation_frames + first = train_and_evaluate_model( + fake_model, + train_df=train, + validation_df=validation, + test_df=test, + target_recall=1.0, + latency_sample_count=1, + ) + + changed_test = test.copy() + changed_test["label"] = list(reversed(changed_test["label"].tolist())) + second_model = type(fake_model)() + second = train_and_evaluate_model( + second_model, + train_df=train, + validation_df=validation, + test_df=changed_test, + target_recall=1.0, + latency_sample_count=1, + ) + + assert first.selected_threshold == second.selected_threshold diff --git a/tests/data_science/SMSModel/evaluation/test_latency.py b/tests/data_science/SMSModel/evaluation/test_latency.py new file mode 100644 index 0000000..5066eff --- /dev/null +++ b/tests/data_science/SMSModel/evaluation/test_latency.py @@ -0,0 +1,69 @@ +"""단건 추론 지연시간 측정 테스트.""" + +import pandas as pd +import pytest + +from data_science.SMSModel.evaluation.latency import ( + measure_single_inference_latency, +) + + +def test_measures_requested_samples_and_warmups(fake_model): + frame = pd.DataFrame( + { + "label": ["normal", "phishing", "normal"], + "mock_score": [0.1, 0.9, 0.2], + } + ) + fake_model.fit(frame) + + metrics = measure_single_inference_latency( + fake_model, + frame, + sample_count=2, + warmup_count=3, + ) + + assert metrics.sample_count == 2 + assert metrics.warmup_count == 3 + assert fake_model.predict_call_count == 5 + assert metrics.minimum_ms >= 0.0 + assert metrics.minimum_ms <= metrics.average_ms <= metrics.maximum_ms + assert metrics.p95_ms >= metrics.median_ms + + +def test_sample_count_is_capped_by_dataframe_size(fake_model): + frame = pd.DataFrame({"mock_score": [0.2, 0.8]}) + fake_model.fit(frame) + result = measure_single_inference_latency( + fake_model, + frame, + sample_count=100, + warmup_count=0, + ) + assert result.sample_count == 2 + + +@pytest.mark.parametrize( + "frame, sample_count, warmup_count, message", + [ + (pd.DataFrame(), 1, 0, "empty"), + (pd.DataFrame({"mock_score": [0.1]}), 0, 0, "greater than 0"), + (pd.DataFrame({"mock_score": [0.1]}), 1, -1, "negative"), + ], +) +def test_rejects_invalid_latency_arguments( + fake_model, + frame, + sample_count, + warmup_count, + message, +): + fake_model.fit(pd.DataFrame({"mock_score": [0.1]})) + with pytest.raises(ValueError, match=message): + measure_single_inference_latency( + fake_model, + frame, + sample_count=sample_count, + warmup_count=warmup_count, + ) diff --git a/tests/data_science/SMSModel/evaluation/test_metrics.py b/tests/data_science/SMSModel/evaluation/test_metrics.py new file mode 100644 index 0000000..989bb1d --- /dev/null +++ b/tests/data_science/SMSModel/evaluation/test_metrics.py @@ -0,0 +1,49 @@ +"""분류 성능 및 혼동행렬 테스트.""" + +import pytest + +from data_science.SMSModel.evaluation.metrics import ( + calculate_classification_metrics, +) + + +def test_calculates_metrics_and_confusion_matrix_in_fixed_order(): + metrics = calculate_classification_metrics( + ["normal", "normal", "phishing", "phishing"], + ["normal", "phishing", "normal", "phishing"], + ) + + assert metrics.sample_count == 4 + assert metrics.true_negative == 1 + assert metrics.false_positive == 1 + assert metrics.false_negative == 1 + assert metrics.true_positive == 1 + assert metrics.precision == pytest.approx(0.5) + assert metrics.recall == pytest.approx(0.5) + assert metrics.f1 == pytest.approx(0.5) + assert metrics.f2 == pytest.approx(0.5) + assert metrics.to_dict()["false_negative"] == 1 + + +def test_handles_zero_positive_predictions_without_division_error(): + metrics = calculate_classification_metrics( + ["normal", "phishing"], + ["normal", "normal"], + ) + assert metrics.precision == 0.0 + assert metrics.recall == 0.0 + assert metrics.false_negative == 1 + + +@pytest.mark.parametrize( + "y_true, y_pred, message", + [ + ([], [], "empty"), + (["normal"], ["normal", "phishing"], "same length"), + (["unknown"], ["normal"], "y_true"), + (["normal"], ["unknown"], "y_pred"), + ], +) +def test_rejects_invalid_metric_inputs(y_true, y_pred, message): + with pytest.raises(ValueError, match=message): + calculate_classification_metrics(y_true, y_pred) diff --git a/tests/data_science/SMSModel/evaluation/test_reporting.py b/tests/data_science/SMSModel/evaluation/test_reporting.py new file mode 100644 index 0000000..dfd03cf --- /dev/null +++ b/tests/data_science/SMSModel/evaluation/test_reporting.py @@ -0,0 +1,54 @@ +"""모델 평가 JSON, CSV, Markdown 출력 테스트.""" + +import csv +import json + +import pytest + +from data_science.SMSModel.evaluation.evaluator import ( + train_and_evaluate_model, +) +from data_science.SMSModel.evaluation.reporting import ( + save_model_evaluation_reports, +) + + +def test_saves_json_csv_and_markdown_reports( + tmp_path, + fake_model, + evaluation_frames, +): + train, validation, test = evaluation_frames + result = train_and_evaluate_model( + fake_model, + train_df=train, + validation_df=validation, + test_df=test, + target_recall=1.0, + latency_sample_count=1, + ) + + save_model_evaluation_reports([result], output_directory=tmp_path) + + json_path = tmp_path / "model_evaluation.json" + csv_path = tmp_path / "model_evaluation.csv" + markdown_path = tmp_path / "model_evaluation.md" + assert json_path.exists() and csv_path.exists() and markdown_path.exists() + + payload = json.loads(json_path.read_text(encoding="utf-8")) + assert payload["schema_version"] == 1 + assert payload["models"][0]["model_name"] == "fake_probability" + + with csv_path.open(encoding="utf-8", newline="") as csv_file: + rows = list(csv.DictReader(csv_file)) + assert rows[0]["false_negative"] == "0" + assert rows[0]["recall"] == "1.0" + + markdown = markdown_path.read_text(encoding="utf-8") + assert "# Phishing Model Evaluation" in markdown + assert "fake_probability" in markdown + + +def test_rejects_empty_report_result_list(tmp_path): + with pytest.raises(ValueError, match="at least one"): + save_model_evaluation_reports([], output_directory=tmp_path) diff --git a/tests/data_science/SMSModel/evaluation/test_threshold.py b/tests/data_science/SMSModel/evaluation/test_threshold.py new file mode 100644 index 0000000..bdf0683 --- /dev/null +++ b/tests/data_science/SMSModel/evaluation/test_threshold.py @@ -0,0 +1,57 @@ +"""Validation threshold 선택 정책 테스트.""" + +import numpy as np +import pytest + +from data_science.SMSModel.evaluation.threshold import ( + select_validation_threshold, +) + + +def test_selects_highest_quality_threshold_meeting_recall_target(): + result = select_validation_threshold( + ["normal", "normal", "phishing", "phishing"], + [0.1, 0.4, 0.6, 0.9], + target_recall=1.0, + ) + + assert result.threshold == pytest.approx(0.6) + assert result.precision == pytest.approx(1.0) + assert result.recall == pytest.approx(1.0) + assert result.f2 == pytest.approx(1.0) + assert result.false_negative_count == 0 + assert result.target_recall_met is True + + +def test_threshold_selection_is_deterministic(): + arguments = ( + ["normal", "phishing", "normal", "phishing"], + [0.2, 0.7, 0.4, 0.8], + ) + assert select_validation_threshold(*arguments) == select_validation_threshold( + *arguments + ) + + +@pytest.mark.parametrize("target", [0.0, -0.1, 1.1]) +def test_rejects_invalid_target_recall(target): + with pytest.raises(ValueError, match="target_recall"): + select_validation_threshold( + ["normal", "phishing"], + [0.1, 0.9], + target_recall=target, + ) + + +@pytest.mark.parametrize( + "labels, scores, message", + [ + ([], [], "empty"), + (["normal"], [0.1, 0.2], "same length"), + (["unknown"], [0.1], "unsupported labels"), + (["normal"], [np.nan], "finite"), + ], +) +def test_rejects_invalid_validation_inputs(labels, scores, message): + with pytest.raises(ValueError, match=message): + select_validation_threshold(labels, scores) diff --git a/tests/data_science/SMSModel/modeling/conftest.py b/tests/data_science/SMSModel/modeling/conftest.py new file mode 100644 index 0000000..0db94c0 --- /dev/null +++ b/tests/data_science/SMSModel/modeling/conftest.py @@ -0,0 +1,30 @@ +"""Shared fixtures for SMS model classifier tests.""" + +import pandas as pd +import pytest + + +@pytest.fixture +def training_dataframe(): + """Return a balanced dataset large enough for calibrated NB tests.""" + rows = [] + + for index in range(20): + rows.append( + { + "text": f"오늘 회의 시간 안내 {index}", + "text_norm": f"오늘 회의 시간 안내 {index}", + "has_url": False, + "label": "normal", + } + ) + rows.append( + { + "text": f"계좌 정지 확인 필요 http://bit.ly/fake{index}", + "text_norm": "계좌 정지 확인 필요 [URL]", + "has_url": True, + "label": "phishing", + } + ) + + return pd.DataFrame(rows) diff --git a/tests/data_science/SMSModel/modeling/test_artifacts.py b/tests/data_science/SMSModel/modeling/test_artifacts.py new file mode 100644 index 0000000..78906f2 --- /dev/null +++ b/tests/data_science/SMSModel/modeling/test_artifacts.py @@ -0,0 +1,127 @@ +"""Naive Bayes 운영 artifact와 기존 API 호환 테스트.""" + +import joblib +import pytest + +from app.analysis.text import naive_bayes_analyzer as api_analyzer +from data_science.SMSModel.modeling import ( + NaiveBayesPhishingClassifier, + save_operational_naive_bayes_artifacts, +) + + +@pytest.fixture(autouse=True) +def reset_api_model_cache(): + """테스트별로 기존 API의 모듈 전역 모델 캐시를 초기화합니다.""" + for name, value in ( + ("_model", None), + ("_vectorizer", None), + ("_threshold", None), + ("_classes", None), + ("_load_error", None), + ("_load_attempted", False), + ): + setattr(api_analyzer, name, value) + yield + api_analyzer._model = None + api_analyzer._vectorizer = None + api_analyzer._threshold = None + api_analyzer._classes = None + api_analyzer._load_error = None + api_analyzer._load_attempted = False + + +def test_structural_artifact_keeps_existing_api_schema( + tmp_path, + training_dataframe, +): + classifier = NaiveBayesPhishingClassifier( + include_structural_features=True, + calibration_cv=2, + ).fit(training_dataframe) + model_path = tmp_path / "model.pkl" + vectorizer_path = tmp_path / "vectorizer.pkl" + + save_operational_naive_bayes_artifacts( + classifier, + threshold=0.4, + model_path=model_path, + vectorizer_path=vectorizer_path, + ) + + artifact = joblib.load(model_path) + vectorizer = joblib.load(vectorizer_path) + assert set(artifact) == {"model", "threshold", "classes"} + assert artifact["threshold"] == 0.4 + assert artifact["classes"] == ["normal", "phishing"] + assert hasattr(artifact["model"], "predict_proba") + assert hasattr(vectorizer, "transform") + + +@pytest.mark.asyncio +async def test_saved_structural_artifact_loads_in_existing_api( + monkeypatch, + tmp_path, + training_dataframe, +): + classifier = NaiveBayesPhishingClassifier( + include_structural_features=True, + calibration_cv=2, + ).fit(training_dataframe) + model_path = tmp_path / "model.pkl" + vectorizer_path = tmp_path / "vectorizer.pkl" + save_operational_naive_bayes_artifacts( + classifier, + threshold=0.4, + model_path=model_path, + vectorizer_path=vectorizer_path, + ) + monkeypatch.setattr(api_analyzer, "MODEL_PATH", model_path) + monkeypatch.setattr(api_analyzer, "VECTORIZER_PATH", vectorizer_path) + + result = await api_analyzer.analyze_text_with_naive_bayes( + "계좌 정지 확인 필요 http://bit.ly/fake99" + ) + + assert result["engine"] == "naive_bayes" + assert result["is_available"] is True + assert 0 <= result["result"]["risk_score"] <= 100 + assert result["result"]["error_message"] is None + + +def test_operational_artifact_rejects_text_only_model( + tmp_path, + training_dataframe, +): + classifier = NaiveBayesPhishingClassifier( + include_structural_features=False, + calibration_cv=2, + ).fit(training_dataframe) + + with pytest.raises(ValueError, match="structural"): + save_operational_naive_bayes_artifacts( + classifier, + threshold=0.4, + model_path=tmp_path / "model.pkl", + vectorizer_path=tmp_path / "vectorizer.pkl", + ) + + +@pytest.mark.parametrize("threshold", [-0.01, 1.01]) +def test_operational_artifact_rejects_invalid_probability_threshold( + tmp_path, + training_dataframe, + threshold, +): + classifier = NaiveBayesPhishingClassifier( + include_structural_features=True, + calibration_cv=2, + ).fit(training_dataframe) + + with pytest.raises(ValueError, match="between 0 and 1"): + save_operational_naive_bayes_artifacts( + classifier, + threshold=threshold, + model_path=tmp_path / "model.pkl", + vectorizer_path=tmp_path / "vectorizer.pkl", + ) diff --git a/tests/data_science/SMSModel/modeling/test_baseline_runner.py b/tests/data_science/SMSModel/modeling/test_baseline_runner.py new file mode 100644 index 0000000..dcb8eb4 --- /dev/null +++ b/tests/data_science/SMSModel/modeling/test_baseline_runner.py @@ -0,0 +1,101 @@ +"""Naive Bayes baseline 실행기의 orchestration 테스트.""" + +from types import SimpleNamespace + +import pandas as pd + +from data_science.SMSModel.evaluation.evaluator import ModelEvaluationResult +from data_science.SMSModel.evaluation.latency import LatencyMetrics +from data_science.SMSModel.evaluation.metrics import ClassificationMetrics +from data_science.SMSModel.evaluation.threshold import ThresholdSelection +from data_science.SMSModel import run_naive_bayes_baseline as runner + + +def make_result(model_name: str, threshold: float) -> ModelEvaluationResult: + return ModelEvaluationResult( + model_name=model_name, + score_type="probability", + selected_threshold=threshold, + validation=ThresholdSelection( + threshold=threshold, + precision=1.0, + recall=1.0, + f2=1.0, + false_negative_count=0, + target_recall=0.96, + target_recall_met=True, + ), + test_metrics=ClassificationMetrics( + sample_count=2, + precision=1.0, + recall=1.0, + f1=1.0, + f2=1.0, + true_negative=1, + false_positive=0, + false_negative=0, + true_positive=1, + ), + latency=LatencyMetrics( + sample_count=1, + warmup_count=0, + average_ms=0.1, + median_ms=0.1, + p95_ms=0.1, + minimum_ms=0.1, + maximum_ms=0.1, + ), + metadata={"model_name": model_name}, + ) + + +def test_runner_evaluates_both_variants_without_deploying_artifact( + monkeypatch, + tmp_path, +): + frame = pd.DataFrame( + { + "text": ["정상", "피싱 [URL]"], + "text_norm": ["정상", "피싱 [URL]"], + "has_url": [False, True], + "label": ["normal", "phishing"], + } + ) + splits = SimpleNamespace(train=frame, validation=frame, test=frame) + evaluated_models = [] + report_calls = [] + + monkeypatch.setattr(runner, "load_data", lambda _path: (frame, frame.iloc[0:0])) + monkeypatch.setattr(runner, "split_data", lambda _df: splits) + + def fake_evaluate(model, **kwargs): + evaluated_models.append((model, kwargs)) + threshold = 0.35 if model.include_structural_features else 0.45 + return make_result(model.model_name, threshold) + + monkeypatch.setattr(runner, "train_and_evaluate_model", fake_evaluate) + monkeypatch.setattr( + runner, + "save_model_evaluation_reports", + lambda results, **kwargs: report_calls.append((results, kwargs)), + ) + monkeypatch.setattr(runner, "MODEL_PATH", tmp_path / "model.pkl") + monkeypatch.setattr(runner, "VECTORIZER_PATH", tmp_path / "vectorizer.pkl") + monkeypatch.setattr(runner, "NB_REPORT_DIRECTORY", tmp_path / "reports") + + results = runner.evaluate_naive_bayes_baselines() + + assert [model.include_structural_features for model, _ in evaluated_models] == [ + False, + True, + ] + assert [result.model_name for result in results] == [ + "naive_bayes_text_only", + "naive_bayes_structural", + ] + for _model, kwargs in evaluated_models: + assert kwargs["train_df"] is frame + assert kwargs["validation_df"] is frame + assert kwargs["test_df"] is frame + assert report_calls[0][0] == results + assert report_calls[0][1]["output_directory"] == tmp_path / "reports" diff --git a/tests/data_science/SMSModel/modeling/test_naive_bayes.py b/tests/data_science/SMSModel/modeling/test_naive_bayes.py new file mode 100644 index 0000000..1bfca0f --- /dev/null +++ b/tests/data_science/SMSModel/modeling/test_naive_bayes.py @@ -0,0 +1,85 @@ +import numpy as np +import pytest + +from data_science.SMSModel.modeling import ( + NaiveBayesPhishingClassifier, + ScoreType, +) + +@pytest.mark.parametrize( + "include_structural_features", + [False, True], +) +def test_naive_bayes_fit_and_predict_scores( + training_dataframe, + include_structural_features, +): + model = NaiveBayesPhishingClassifier( + include_structural_features=( + include_structural_features + ), + calibration_cv=2, + ) + + model.fit(training_dataframe) + + scores = model.predict_scores( + training_dataframe.iloc[:4] + ) + + assert scores.score_type == ( + ScoreType.PROBABILITY + ) + assert scores.values.shape == (4,) + assert np.all(scores.values >= 0.0) + assert np.all(scores.values <= 1.0) + + +def test_structural_model_has_six_additional_features( + training_dataframe, +): + text_only = NaiveBayesPhishingClassifier( + include_structural_features=False, + calibration_cv=2, + ) + structural = NaiveBayesPhishingClassifier( + include_structural_features=True, + calibration_cv=2, + ) + + text_only.fit(training_dataframe) + structural.fit(training_dataframe) + + text_only_matrix = ( + text_only._build_feature_matrix( + training_dataframe, + fit_vectorizer=False, + ) + ) + structural_matrix = ( + structural._build_feature_matrix( + training_dataframe, + fit_vectorizer=False, + ) + ) + + assert ( + structural_matrix.shape[1] + == text_only_matrix.shape[1] + 6 + ) + + +def test_predict_before_fit_fails( + training_dataframe, +): + model = NaiveBayesPhishingClassifier( + include_structural_features=True, + ) + + with pytest.raises( + RuntimeError, + match="not fitted", + ): + model.predict_scores( + training_dataframe + ) diff --git a/tests/data_science/SMSModel/test_dataset_split_report.py b/tests/data_science/SMSModel/test_dataset_split_report.py new file mode 100644 index 0000000..515989c --- /dev/null +++ b/tests/data_science/SMSModel/test_dataset_split_report.py @@ -0,0 +1,165 @@ +"""데이터 분할 통계 보고서 테스트.""" + +import json + +import pandas as pd +import pytest + +from data_science.SMSModel.dataset_splitting import ( + DatasetSplitConfig, + DatasetSplits, + split_grouped_dataset, +) +from data_science.SMSModel.reporting import ( + build_dataset_split_summary, + calculate_dataset_fingerprint, + generate_dataset_split_reports, +) +from data_science.SMSModel.template_grouping import TemplateGroupingConfig + + +@pytest.fixture +def report_data(): + rows = [] + message_types = ("기관사칭", "택배사칭", "정상알림톡") + for group_index in range(60): + label = "phishing" if group_index % 2 == 0 else "normal" + for member_index in range(2 if group_index % 6 == 0 else 1): + rows.append( + { + "text_fingerprint": f"fp-{group_index}-{member_index}", + "template_group_id": f"group-{group_index}", + "label": label, + "type": message_types[group_index % len(message_types)], + } + ) + source = pd.DataFrame(rows) + split_config = DatasetSplitConfig(candidate_count=100) + grouping_config = TemplateGroupingConfig( + similarity_threshold=0.88, + ngram_range=(2, 5), + ) + splits = split_grouped_dataset(source, config=split_config) + return source, splits, split_config, grouping_config + + +def test_dataset_fingerprint_is_reproducible(report_data): + source, _, _, _ = report_data + assert calculate_dataset_fingerprint(source) == calculate_dataset_fingerprint( + source.copy() + ) + + +def test_dataset_fingerprint_ignores_row_order(report_data): + source, _, _, _ = report_data + shuffled = source.sample(frac=1.0, random_state=99).reset_index(drop=True) + assert calculate_dataset_fingerprint(source) == calculate_dataset_fingerprint( + shuffled + ) + + +def test_dataset_fingerprint_changes_when_label_changes(report_data): + source, _, _, _ = report_data + changed = source.copy() + changed.loc[0, "label"] = "normal" + assert calculate_dataset_fingerprint(source) != calculate_dataset_fingerprint( + changed + ) + + +def test_summary_contains_counts_distributions_and_configuration(report_data): + source, splits, split_config, grouping_config = report_data + summary = build_dataset_split_summary( + source, + splits, + split_config=split_config, + grouping_config=grouping_config, + ) + + assert summary["dataset"]["row_count"] == len(source) + assert summary["splits"]["train"]["row_count"] == len(splits.train) + assert set(summary["splits"]["test"]["labels"]) == { + "normal", + "phishing", + } + assert summary["splits"]["validation"]["types"] + assert summary["configuration"]["template_grouping"][ + "similarity_threshold" + ] == 0.88 + assert summary["validation"]["group_overlap_count"] == 0 + assert summary["validation"]["fingerprint_overlap_count"] == 0 + + +def test_report_generation_creates_valid_json_and_markdown( + tmp_path, + report_data, +): + source, splits, split_config, grouping_config = report_data + json_path = tmp_path / "summary.json" + markdown_path = tmp_path / "summary.md" + + generate_dataset_split_reports( + source, + splits, + split_config=split_config, + grouping_config=grouping_config, + json_path=json_path, + markdown_path=markdown_path, + ) + + parsed = json.loads(json_path.read_text(encoding="utf-8")) + markdown = markdown_path.read_text(encoding="utf-8") + assert parsed["validation"]["passed"] is True + assert "## Split Overview" in markdown + assert "## Message Type Distribution" in markdown + + +def test_report_generation_is_deterministic(tmp_path, report_data): + source, splits, split_config, grouping_config = report_data + first_json = tmp_path / "first.json" + first_md = tmp_path / "first.md" + second_json = tmp_path / "second.json" + second_md = tmp_path / "second.md" + + for json_path, markdown_path in ( + (first_json, first_md), + (second_json, second_md), + ): + generate_dataset_split_reports( + source, + splits, + split_config=split_config, + grouping_config=grouping_config, + json_path=json_path, + markdown_path=markdown_path, + ) + + assert first_json.read_bytes() == second_json.read_bytes() + assert first_md.read_bytes() == second_md.read_bytes() + + +def test_report_generation_stops_on_group_leakage(tmp_path, report_data): + source, splits, split_config, grouping_config = report_data + invalid_splits = DatasetSplits( + train=splits.train.copy(), + validation=splits.validation.copy(), + test=splits.test.copy(), + ) + invalid_splits.validation.loc[0, "template_group_id"] = ( + invalid_splits.train.iloc[0]["template_group_id"] + ) + json_path = tmp_path / "summary.json" + markdown_path = tmp_path / "summary.md" + + with pytest.raises(ValueError, match="template group leakage"): + generate_dataset_split_reports( + source, + invalid_splits, + split_config=split_config, + grouping_config=grouping_config, + json_path=json_path, + markdown_path=markdown_path, + ) + + assert not json_path.exists() + assert not markdown_path.exists() diff --git a/tests/data_science/SMSModel/test_dataset_splitting.py b/tests/data_science/SMSModel/test_dataset_splitting.py new file mode 100644 index 0000000..78a5e4b --- /dev/null +++ b/tests/data_science/SMSModel/test_dataset_splitting.py @@ -0,0 +1,111 @@ +"""그룹 보존 split과 manifest의 단위 테스트.""" + +import pandas as pd +import pytest + +from data_science.SMSModel.dataset_splitting import ( + DatasetSplitConfig, + load_split_manifest, + save_split_manifest, + split_grouped_dataset, + validate_dataset_splits, +) + + +def make_dataset(group_count: int = 60) -> pd.DataFrame: + rows = [] + for group_index in range(group_count): + label = "phishing" if group_index % 2 == 0 else "normal" + # 일부 그룹은 두 행을 가져 그룹 단위 분할을 실제로 검증합니다. + member_count = 2 if group_index % 5 == 0 else 1 + for member_index in range(member_count): + rows.append( + { + "text_fingerprint": f"fp-{group_index}-{member_index}", + "template_group_id": f"group-{group_index}", + "label": label, + "type": "test-type", + } + ) + return pd.DataFrame(rows) + + +def fingerprint_sets(splits): + return { + "train": set(splits.train["text_fingerprint"]), + "validation": set(splits.validation["text_fingerprint"]), + "test": set(splits.test["text_fingerprint"]), + } + + +def test_same_template_group_stays_in_one_split(): + splits = split_grouped_dataset(make_dataset()) + combined = pd.concat([splits.train, splits.validation, splits.test]) + assert combined.groupby("template_group_id")["split"].nunique().max() == 1 + + +def test_split_is_reproducible_with_same_seed(): + df = make_dataset() + first = split_grouped_dataset(df, config=DatasetSplitConfig(random_state=42)) + second = split_grouped_dataset(df, config=DatasetSplitConfig(random_state=42)) + assert fingerprint_sets(first) == fingerprint_sets(second) + + +def test_split_ratios_are_close_to_targets(): + df = make_dataset() + splits = split_grouped_dataset(df) + assert abs(len(splits.train) / len(df) - 0.70) <= 0.05 + assert abs(len(splits.validation) / len(df) - 0.15) <= 0.05 + assert abs(len(splits.test) / len(df) - 0.15) <= 0.05 + + +def test_each_split_contains_both_labels(): + splits = split_grouped_dataset(make_dataset()) + for part in (splits.train, splits.validation, splits.test): + assert set(part["label"]) == {"normal", "phishing"} + + +def test_validation_rejects_group_leakage(): + df = make_dataset() + splits = split_grouped_dataset(df) + splits.validation.loc[0, "template_group_id"] = splits.train.iloc[0][ + "template_group_id" + ] + with pytest.raises(ValueError, match="template group leakage"): + validate_dataset_splits(df, splits) + + +def test_manifest_round_trip(tmp_path): + df = make_dataset() + expected = split_grouped_dataset(df) + path = tmp_path / "split.csv" + save_split_manifest(expected, path) + actual = load_split_manifest(df, path) + assert fingerprint_sets(actual) == fingerprint_sets(expected) + + +def test_manifest_rejects_changed_dataset(tmp_path): + df = make_dataset() + path = tmp_path / "split.csv" + save_split_manifest(split_grouped_dataset(df), path) + changed = df.iloc[:-1].copy() + with pytest.raises(ValueError, match="does not match"): + load_split_manifest(changed, path) + + +def test_manifest_rejects_changed_group_assignment(tmp_path): + df = make_dataset() + path = tmp_path / "split.csv" + save_split_manifest(split_grouped_dataset(df), path) + changed = df.copy() + changed.loc[0, "template_group_id"] = "changed-group" + with pytest.raises(ValueError, match="template_group_id"): + load_split_manifest(changed, path) + + +def test_manifest_does_not_overwrite_by_default(tmp_path): + splits = split_grouped_dataset(make_dataset()) + path = tmp_path / "split.csv" + save_split_manifest(splits, path) + with pytest.raises(FileExistsError): + save_split_manifest(splits, path) diff --git a/tests/data_science/SMSModel/test_template_grouping.py b/tests/data_science/SMSModel/test_template_grouping.py new file mode 100644 index 0000000..f240794 --- /dev/null +++ b/tests/data_science/SMSModel/test_template_grouping.py @@ -0,0 +1,319 @@ +import pandas as pd +import pytest + +from data_science.SMSModel.template_grouping import ( + TemplateGroupingConfig, + add_text_fingerprints, + assign_template_groups, + create_text_fingerprint, + prepare_template_groups, + remove_exact_duplicates, + validate_duplicate_labels, +) + + +def make_dataframe( + rows: list[tuple[str, str]], +) -> pd.DataFrame: + + return pd.DataFrame( + [ + { + "text_norm": text_norm, + "label": label, + } + for text_norm, label in rows + ] + ) + + +def test_create_text_fingerprint_is_deterministic(): + first = create_text_fingerprint( + "[URL]에서 건강검진 결과를 확인하세요" + ) + second = create_text_fingerprint( + "[URL]에서 건강검진 결과를 확인하세요" + ) + + assert first == second + assert len(first) == 64 + + +def test_create_text_fingerprint_ignores_outer_whitespace(): + plain = create_text_fingerprint("본인 인증이 필요합니다") + padded = create_text_fingerprint( + " 본인 인증이 필요합니다 " + ) + + assert plain == padded + + +def test_different_texts_have_different_fingerprints(): + first = create_text_fingerprint("배송 조회가 필요합니다") + second = create_text_fingerprint("결제 확인이 필요합니다") + + assert first != second + + +def test_add_text_fingerprints_does_not_modify_original_dataframe(): + original = make_dataframe( + [ + ("배송 조회가 필요합니다", "normal"), + ] + ) + + result = add_text_fingerprints(original) + + assert "text_fingerprint" not in original.columns + assert "text_fingerprint" in result.columns + + +def test_remove_exact_duplicates_keeps_one_row(): + df = make_dataframe( + [ + ("[URL]에서 본인 인증하세요", "phishing"), + ("[URL]에서 본인 인증하세요", "phishing"), + ("회의는 오후 세 시입니다", "normal"), + ] + ) + df = add_text_fingerprints(df) + + result = remove_exact_duplicates(df) + + assert len(result) == 2 + assert result["text_norm"].tolist() == [ + "[URL]에서 본인 인증하세요", + "회의는 오후 세 시입니다", + ] + + +def test_validate_duplicate_labels_rejects_conflicting_labels(): + df = make_dataframe( + [ + ("[URL]에서 본인 인증하세요", "phishing"), + ("[URL]에서 본인 인증하세요", "normal"), + ] + ) + df = add_text_fingerprints(df) + + with pytest.raises( + ValueError, + match="conflicting labels", + ): + validate_duplicate_labels(df) + + +def test_similar_messages_receive_same_template_group(): + df = make_dataframe( + [ + ( + "[국민건강보험] 건강검진 결과가 발급되었습니다 " + "[URL]에서 즉시 확인하세요", + "phishing", + ), + ( + "[국민건강보험] 건강검진 보고서가 발급되었습니다 " + "[URL]에서 지금 확인하세요", + "phishing", + ), + ( + "오늘 저녁 식사는 일곱 시에 시작합니다", + "normal", + ), + ] + ) + + config = TemplateGroupingConfig( + similarity_threshold=0.70, + ngram_range=(2, 4), + min_df=1, + max_features=None, + ) + + result = prepare_template_groups( + df, + config=config, + ) + + first_group = result.iloc[0]["template_group_id"] + second_group = result.iloc[1]["template_group_id"] + unrelated_group = result.iloc[2]["template_group_id"] + + assert first_group == second_group + assert first_group != unrelated_group + + +def test_unrelated_messages_receive_different_template_groups(): + df = make_dataframe( + [ + ( + "[URL]에서 택배 배송 주소를 확인하세요", + "phishing", + ), + ( + "회의가 오후 세 시로 변경되었습니다", + "normal", + ), + ( + "저녁 식사 재료를 구매했습니다", + "normal", + ), + ] + ) + + config = TemplateGroupingConfig( + similarity_threshold=0.88, + ) + + result = prepare_template_groups( + df, + config=config, + ) + + assert result["template_group_id"].nunique() == 3 + + +def test_group_id_is_stable_when_row_order_changes(): + rows = [ + ( + "[국민건강보험] 건강검진 결과가 발급되었습니다 " + "[URL]에서 즉시 확인하세요", + "phishing", + ), + ( + "[국민건강보험] 건강검진 보고서가 발급되었습니다 " + "[URL]에서 지금 확인하세요", + "phishing", + ), + ( + "오늘 저녁 식사는 일곱 시에 시작합니다", + "normal", + ), + ] + + config = TemplateGroupingConfig( + similarity_threshold=0.70, + ngram_range=(2, 4), + ) + + original = prepare_template_groups( + make_dataframe(rows), + config=config, + ) + + reversed_result = prepare_template_groups( + make_dataframe(list(reversed(rows))), + config=config, + ) + + original_mapping = dict( + zip( + original["text_norm"], + original["template_group_id"], + strict=True, + ) + ) + reversed_mapping = dict( + zip( + reversed_result["text_norm"], + reversed_result["template_group_id"], + strict=True, + ) + ) + + assert original_mapping == reversed_mapping + + +def test_transitively_similar_messages_are_grouped_together( + monkeypatch, +): + df = make_dataframe( + [ + ("메시지 A", "phishing"), + ("메시지 B", "phishing"), + ("메시지 C", "phishing"), + ] + ) + df = add_text_fingerprints(df) + + monkeypatch.setattr( + "data_science.SMSModel.template_grouping.similarity._find_similar_pairs", + lambda texts, config: [ + (0, 1, 0.90), + (1, 2, 0.91), + ], + ) + + result = assign_template_groups(df) + + assert result["template_group_id"].nunique() == 1 + + +def test_prepare_template_groups_adds_required_columns(): + df = make_dataframe( + [ + ("배송 주소 확인 [URL]", "phishing"), + ("평범한 일상 대화입니다", "normal"), + ] + ) + + result = prepare_template_groups(df) + + assert "text_fingerprint" in result.columns + assert "template_group_id" in result.columns + assert result["text_fingerprint"].notna().all() + assert result["template_group_id"].notna().all() + + +def test_prepare_template_groups_handles_empty_dataframe(): + df = pd.DataFrame( + { + "text_norm": pd.Series(dtype="string"), + "label": pd.Series(dtype="string"), + } + ) + + result = prepare_template_groups(df) + + assert result.empty + assert "text_fingerprint" in result.columns + assert "template_group_id" in result.columns + + +@pytest.mark.parametrize( + "threshold", + [ + 0.0, + -0.1, + 1.1, + ], +) +def test_grouping_config_rejects_invalid_threshold( + threshold: float, +): + with pytest.raises( + ValueError, + match="similarity_threshold", + ): + TemplateGroupingConfig( + similarity_threshold=threshold, + ) + +@pytest.mark.slow +def test_real_dataset_template_grouping_smoke(): + + from data_science.SMSModel.train_sms import DATA_PATH, load_data + + df_pool, df_holdout = load_data(DATA_PATH) + + assert not df_pool.empty + assert "text_norm" in df_pool.columns + assert "text_fingerprint" in df_pool.columns + assert "template_group_id" in df_pool.columns + + assert df_pool["text_fingerprint"].is_unique + assert df_pool["template_group_id"].notna().all() + + assert set(df_pool["label"].unique()).issubset( + {"normal", "phishing"} + ) diff --git a/tests/data_science/SMSModel/tokenization/__init__.py b/tests/data_science/SMSModel/tokenization/__init__.py new file mode 100644 index 0000000..81818ab --- /dev/null +++ b/tests/data_science/SMSModel/tokenization/__init__.py @@ -0,0 +1 @@ +"""SMS 형태소 tokenizer 테스트 패키지""" \ No newline at end of file diff --git a/tests/data_science/SMSModel/tokenization/test_kiwi_tokenizer.py b/tests/data_science/SMSModel/tokenization/test_kiwi_tokenizer.py new file mode 100644 index 0000000..65e2377 --- /dev/null +++ b/tests/data_science/SMSModel/tokenization/test_kiwi_tokenizer.py @@ -0,0 +1,198 @@ +"""Kiwi 기반 SMS tokenizer 단위 테스트""" + +import joblib +import pytest +from sklearn.feature_extraction.text import CountVectorizer + +from data_science.SMSModel.tokenization import ( + MASK_TOKENS, + SELECTED_POS_TAGS, + kiwi_tokenize, +) +from data_science.SMSModel.tokenization.kiwi_tokenizer import ( + _base_pos_tag, + _get_kiwi, +) + + +def test_tokenize_korean_phishing_message(): + tokens = kiwi_tokenize( + "계좌가 정지되었으니 즉시 본인 인증을 진행하세요." + ) + + assert "계좌" in tokens + assert "정지" in tokens + assert "즉시" in tokens + assert "인증" in tokens + assert "진행" in tokens + + +def test_tokenize_normal_message(): + tokens = kiwi_tokenize( + "오늘 저녁에 같이 식사할까요?" + ) + + assert "오늘" in tokens + assert "저녁" in tokens + assert "같이" in tokens + assert "식사" in tokens + + +@pytest.mark.parametrize( + "text", + [ + "", + " ", + "\n\t", + ], +) +def test_empty_or_whitespace_input_returns_empty_list(text): + assert kiwi_tokenize(text) == [] + + +@pytest.mark.parametrize( + "text", + [ + "!!!", + "...", + "()[]{}", + "😊🚨", + ], +) +def test_special_character_only_input_returns_empty_list(text): + assert kiwi_tokenize(text) == [] + + +@pytest.mark.parametrize( + "value", + [ + None, + 123, + [], + {}, + ], +) +def test_non_string_input_raises_type_error(value): + with pytest.raises( + TypeError, + match="text must be a string", + ): + kiwi_tokenize(value) + + +@pytest.mark.parametrize( + "mask_token", + sorted(MASK_TOKENS), +) +def test_mask_token_is_preserved(mask_token): + tokens = kiwi_tokenize( + f"확인이 필요합니다 {mask_token}" + ) + + assert tokens.count(mask_token) == 1 + + +def test_multiple_mask_tokens_are_preserved_in_order(): + tokens = kiwi_tokenize( + "[URL]에서 [ACCOUNT] 정보를 확인하세요." + ) + + url_index = tokens.index("[URL]") + account_index = tokens.index("[ACCOUNT]") + + assert url_index < account_index + + +def test_particle_and_ending_are_removed(): + text = "고객님의 계좌가 정지되었습니다." + result = kiwi_tokenize(text) + + raw_tokens = _get_kiwi().tokenize( + text, + normalize_coda=True, + ) + excluded_forms = { + token.form + for token in raw_tokens + if _base_pos_tag(token.tag).startswith(("J", "E")) + } + + assert excluded_forms.isdisjoint(result) + + +def test_selected_pos_tags_are_explicitly_defined(): + assert SELECTED_POS_TAGS == { + "NNG", + "NNP", + "NNB", + "NR", + "NP", + "VV", + "VA", + "VX", + "VCP", + "VCN", + "MM", + "MAG", + "MAJ", + "XR", + "SL", + } + + +def test_repeated_calls_return_same_tokens(): + text = "긴급 대출 신청을 위해 본인 인증이 필요합니다." + + first = kiwi_tokenize(text) + second = kiwi_tokenize(text) + + assert first == second + + +def test_kiwi_instance_is_reused(): + assert _get_kiwi() is _get_kiwi() + + +def test_irregular_pos_suffix_is_normalized(): + """Kiwi irregular-conjugation suffixes must not bypass POS filtering.""" + assert _base_pos_tag("VV-I") == "VV" + assert _base_pos_tag("VA-R") == "VA" + assert _base_pos_tag("NNG") == "NNG" + + +def test_tokenizer_integrates_with_count_vectorizer(): + """The public tokenizer must work as a scikit-learn callable.""" + vectorizer = CountVectorizer( + tokenizer=kiwi_tokenize, + token_pattern=None, + lowercase=False, + ) + + matrix = vectorizer.fit_transform( + [ + "계좌가 정지되었습니다 [URL]", + "오늘 저녁 식사 약속입니다", + ] + ) + + assert matrix.shape[0] == 2 + assert "[URL]" in vectorizer.vocabulary_ + assert "계좌" in vectorizer.vocabulary_ + + +def test_vectorizer_with_tokenizer_survives_joblib_round_trip(tmp_path): + """A trained vectorizer must remain usable after artifact serialization.""" + vectorizer = CountVectorizer( + tokenizer=kiwi_tokenize, + token_pattern=None, + lowercase=False, + ) + vectorizer.fit(["본인 인증이 필요합니다 [URL]"]) + artifact_path = tmp_path / "kiwi_vectorizer.pkl" + + joblib.dump(vectorizer, artifact_path) + restored = joblib.load(artifact_path) + + original = vectorizer.transform(["본인 인증 [URL]"]) + reloaded = restored.transform(["본인 인증 [URL]"]) + assert (original != reloaded).nnz == 0 diff --git a/tests/data_science/__init__.py b/tests/data_science/__init__.py new file mode 100644 index 0000000..e69de29