From c07b618350c1477f009f6354eafa863d24cafe92 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 11:34:34 +0300 Subject: [PATCH 1/6] some re-org of dirs/modules --- .github/workflows/collect_commits.yml | 18 ------------------ src/{ => api/forgejo}/ForgejoRepoAPI.py | 0 src/{ => api/github}/GitHubRepoAPI.py | 0 .../github}/graphql/pull_request_parser.py | 0 src/git_logger.py | 2 +- src/interface_wrapper.py | 8 ++++---- 6 files changed, 5 insertions(+), 23 deletions(-) delete mode 100644 .github/workflows/collect_commits.yml rename src/{ => api/forgejo}/ForgejoRepoAPI.py (100%) rename src/{ => api/github}/GitHubRepoAPI.py (100%) rename src/{ => api/github}/graphql/pull_request_parser.py (100%) diff --git a/.github/workflows/collect_commits.yml b/.github/workflows/collect_commits.yml deleted file mode 100644 index cc50997a..00000000 --- a/.github/workflows/collect_commits.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Collect Commits -on: - push: - -jobs: - collect-commits: - runs-on: [self-hosted, mse] - steps: - - name: Collect info - run: | - cd /data/ - mkdir -p '${{github.repository}}' - - cat > ./run.json < str: class RepositoryFactory: @staticmethod def create_api(token: str, base_url: str | None = None) -> IRepositoryAPI: - from src.ForgejoRepoAPI import ForgejoRepoAPI - from src.GitHubRepoAPI import GitHubRepoAPI + from src.api.forgejo.ForgejoRepoAPI import ForgejoRepoAPI + from src.api.github.GitHubRepoAPI import GitHubRepoAPI errors = [] From f91b1e76bd876f845232e1a21c538050ee68afde Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 12:05:12 +0300 Subject: [PATCH 2/6] re-org api wrapper modules (and upd imports) --- src/{interface_wrapper.py => api/baseAPI.py} | 152 +------------------ src/api/forgejo/ForgejoRepoAPI.py | 24 +-- src/api/github/GitHubRepoAPI.py | 40 ++--- src/api/models.py | 127 ++++++++++++++++ src/commits_parser.py | 3 +- src/contributors_parser.py | 3 +- src/git_logger.py | 6 +- src/invites_parser.py | 3 +- src/issues_parser.py | 3 +- src/pull_requests_parser.py | 3 +- src/workflow_runs_parser.py | 3 +- 11 files changed, 167 insertions(+), 200 deletions(-) rename src/{interface_wrapper.py => api/baseAPI.py} (53%) create mode 100644 src/api/models.py diff --git a/src/interface_wrapper.py b/src/api/baseAPI.py similarity index 53% rename from src/interface_wrapper.py rename to src/api/baseAPI.py index 58e0af61..0b8145c8 100644 --- a/src/interface_wrapper.py +++ b/src/api/baseAPI.py @@ -1,139 +1,15 @@ -import logging from abc import ABC, abstractmethod -from dataclasses import dataclass -from datetime import datetime from github import Auth, Github from pyforgejo import PyforgejoApi -# Настройка логирования -logging.basicConfig( - level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" -) - - -# Модельные классы -@dataclass -class Contributor: - username: str - email: str - - -@dataclass -class User: - _id: int - login: str - username: str - email: str - html_url: str - node_id: str - type: str - bio: str - site_admin: bool - - -@dataclass -class Commit: - _id: str - message: str - author: User - date: datetime - files: list[str] - additions: int - deletions: int - - -@dataclass -class Branch: - name: str - last_commit: Commit | None - - -@dataclass -class Repository: - _id: str - name: str - url: str - default_branch: Branch - owner: User - - -@dataclass -class Issue: - _id: int - number: int - title: str - state: str - created_at: datetime - closed_at: datetime - body: str - user: User - closed_by: User - labels: list[str] - milestone: str - - -@dataclass -class PullRequest: - _id: int - title: str - author: User - state: str - created_at: datetime - head_label: str - base_label: str - head_ref: str - base_ref: str - merged_by: User - merged: bool - files: list[str] - issue_url: str - labels: list[str] - milestone: str - comments: int = 0 - review_comments: int = 0 - - -@dataclass -class Invite: - _id: int - invitee: User - created_at: datetime | None - html_url: str - - -@dataclass -class Comment: - body: str - created_at: datetime - author: User - - -@dataclass -class WikiPage: - title: str - content: str - - -@dataclass -class WorkflowRun: - display_title: str - event: str - head_branch: str - head_sha: str - name: str - path: str - created_at: datetime - run_started_at: datetime - updated_at: datetime - conclusion: str - status: str - url: str +from src.api.models import (Branch, Comment, Commit, Contributor, Invite, + Issue, PullRequest, Repository, User, WikiPage, + WorkflowRun) # Интерфейс API class IRepositoryAPI(ABC): - @abstractmethod def get_user_data(self, user) -> User: pass @@ -224,25 +100,3 @@ def create_api(token: str, base_url: str | None = None) -> IRepositoryAPI: errors.append(f"Forgejo login failed: {e}") raise Exception(" / ".join(errors)) - - -# Сервис для расчёта метрик -class CommitmentCalculator: - def __init__(self, api: IRepositoryAPI): - self.api = api - - def calculate(self, repo: Repository) -> dict[str, int]: - if not repo: - return {} - try: - commits = self.api.get_commits(repo) - if not commits: - return {} - result = {} - for commit in commits: - author = commit.author.username - result[author] = result.get(author, 0) + 1 - return result - except Exception as e: - logging.error(f"Failed to calculate commitment for repo {repo.name}: {e}") - return {} diff --git a/src/api/forgejo/ForgejoRepoAPI.py b/src/api/forgejo/ForgejoRepoAPI.py index d76fb937..9186187d 100644 --- a/src/api/forgejo/ForgejoRepoAPI.py +++ b/src/api/forgejo/ForgejoRepoAPI.py @@ -1,27 +1,15 @@ import base64 -import sys import logging +import sys import isodate from pyforgejo import PyforgejoApi -from src.utils import ( - log_exceptions, -) -from src.interface_wrapper import ( - Branch, - Comment, - Commit, - Contributor, - Invite, - IRepositoryAPI, - Issue, - PullRequest, - Repository, - User, - WikiPage, - WorkflowRun -) +from src.api.baseAPI import IRepositoryAPI +from src.api.models import (Branch, Comment, Commit, Contributor, Invite, + Issue, PullRequest, Repository, User, WikiPage, + WorkflowRun) +from src.utils import log_exceptions class ForgejoRepoAPI(IRepositoryAPI): diff --git a/src/api/github/GitHubRepoAPI.py b/src/api/github/GitHubRepoAPI.py index ca87a169..d9be9f11 100644 --- a/src/api/github/GitHubRepoAPI.py +++ b/src/api/github/GitHubRepoAPI.py @@ -1,22 +1,10 @@ from github import Github -from src.utils import ( - log_exceptions, -) -from src.interface_wrapper import ( - Branch, - Comment, - Commit, - Contributor, - Invite, - IRepositoryAPI, - Issue, - PullRequest, - Repository, - User, - WikiPage, - WorkflowRun, -) +from src.api.baseAPI import IRepositoryAPI +from src.api.models import (Branch, Comment, Commit, Contributor, Invite, + Issue, PullRequest, Repository, User, WikiPage, + WorkflowRun) +from src.utils import log_exceptions class GitHubRepoAPI(IRepositoryAPI): @@ -83,7 +71,7 @@ def get_contributors(self, repo: Repository) -> list[Contributor]: @log_exceptions(default_return=[], message="Failed to get issues from GitHub") def get_issues(self, repo: Repository) -> list[Issue]: - issues = self.client.get_repo(repo._id).get_issues(state='all') + issues = self.client.get_repo(repo._id).get_issues(state="all") return [ Issue( _id=i.number, @@ -101,9 +89,11 @@ def get_issues(self, repo: Repository) -> list[Issue]: for i in issues ] - @log_exceptions(default_return=[], message="Failed to get pull requests from GitHub") + @log_exceptions( + default_return=[], message="Failed to get pull requests from GitHub" + ) def get_pull_requests(self, repo: Repository) -> list[PullRequest]: - pulls = self.client.get_repo(repo._id).get_pulls(state='all') + pulls = self.client.get_repo(repo._id).get_pulls(state="all") return [ PullRequest( _id=p.number, @@ -139,7 +129,7 @@ def get_branches(self, repo: Repository) -> list[Branch]: return result def get_wiki_pages(self, repo: Repository) -> list[WikiPage]: - raise Exception('not implemented') + raise Exception("not implemented") @log_exceptions(default_return=[], message="Failed to get forks from GitHub") def get_forks(self, repo: Repository) -> list[Repository]: @@ -200,7 +190,9 @@ def get_invites(self, repo: Repository) -> list[Invite]: def get_rate_limiting(self) -> tuple[int, int]: return self.client.rate_limiting - @log_exceptions(default_return=[], message="Failed to get workflow runs from GitHub") + @log_exceptions( + default_return=[], message="Failed to get workflow runs from GitHub" + ) def get_workflow_runs(self, repo) -> list[WorkflowRun]: runs = self.client.get_repo(repo._id).get_workflow_runs() return [ @@ -222,14 +214,14 @@ def get_workflow_runs(self, repo) -> list[WorkflowRun]: ] def get_base_url(self) -> str: - return 'https://api.github.com' + return "https://api.github.com" # Точка входа для тестирования if __name__ == "__main__": # Создайте клиент GitHub (используйте ваш токен) # client = Github("") - api = GitHubRepoAPI('client') + api = GitHubRepoAPI("client") # Укажите ваш репозиторий repo_name = "" diff --git a/src/api/models.py b/src/api/models.py new file mode 100644 index 00000000..5d442a29 --- /dev/null +++ b/src/api/models.py @@ -0,0 +1,127 @@ +import logging +from dataclasses import dataclass +from datetime import datetime + +# Настройка логирования +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) + + +# Модельные классы +@dataclass +class Contributor: + username: str + email: str + + +@dataclass +class User: + _id: int + login: str + username: str + email: str + html_url: str + node_id: str + type: str + bio: str + site_admin: bool + + +@dataclass +class Commit: + _id: str + message: str + author: User + date: datetime + files: list[str] + additions: int + deletions: int + + +@dataclass +class Branch: + name: str + last_commit: Commit | None + + +@dataclass +class Repository: + _id: str + name: str + url: str + default_branch: Branch + owner: User + + +@dataclass +class Issue: + _id: int + number: int + title: str + state: str + created_at: datetime + closed_at: datetime + body: str + user: User + closed_by: User + labels: list[str] + milestone: str + + +@dataclass +class PullRequest: + _id: int + title: str + author: User + state: str + created_at: datetime + head_label: str + base_label: str + head_ref: str + base_ref: str + merged_by: User + merged: bool + files: list[str] + issue_url: str + labels: list[str] + milestone: str + comments: int = 0 + review_comments: int = 0 + + +@dataclass +class Invite: + _id: int + invitee: User + created_at: datetime | None + html_url: str + + +@dataclass +class Comment: + body: str + created_at: datetime + author: User + + +@dataclass +class WikiPage: + title: str + content: str + + +@dataclass +class WorkflowRun: + display_title: str + event: str + head_branch: str + head_sha: str + name: str + path: str + created_at: datetime + run_started_at: datetime + updated_at: datetime + conclusion: str + status: str + url: str diff --git a/src/commits_parser.py b/src/commits_parser.py index 74a2d8c4..20843215 100644 --- a/src/commits_parser.py +++ b/src/commits_parser.py @@ -5,8 +5,9 @@ import pytz +from src.api.baseAPI import IRepositoryAPI from src.constants import EMPTY_FIELD, GOOGLE_MAX_CELL_LEN, TIMEDELTA, TIMEZONE -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger diff --git a/src/contributors_parser.py b/src/contributors_parser.py index e87fc824..e04c9214 100644 --- a/src/contributors_parser.py +++ b/src/contributors_parser.py @@ -2,8 +2,9 @@ from time import sleep from typing import Generator +from src.api.baseAPI import IRepositoryAPI from src.constants import EMPTY_FIELD, TIMEDELTA -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger diff --git a/src/git_logger.py b/src/git_logger.py index 6c3a5b5f..98d6888c 100644 --- a/src/git_logger.py +++ b/src/git_logger.py @@ -5,10 +5,10 @@ import requests +from src.api.baseAPI import IRepositoryAPI from src.api.github.GitHubRepoAPI import GitHubRepoAPI -from src.interface_wrapper import ( - RepositoryFactory, - IRepositoryAPI +from src.api.baseAPI import ( + RepositoryFactory ) from src.constants import ( EMPTY_FIELD, diff --git a/src/invites_parser.py b/src/invites_parser.py index f56ad778..503975d4 100644 --- a/src/invites_parser.py +++ b/src/invites_parser.py @@ -2,8 +2,9 @@ from time import sleep from typing import Generator +from src.api.baseAPI import IRepositoryAPI from src.constants import TIMEDELTA -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger diff --git a/src/issues_parser.py b/src/issues_parser.py index 1d5d673f..dc199db8 100644 --- a/src/issues_parser.py +++ b/src/issues_parser.py @@ -6,9 +6,10 @@ import pytz import requests +from src.api.baseAPI import IRepositoryAPI from src.constants import EMPTY_FIELD, TIMEDELTA, TIMEZONE from src.git_logger import get_assignee_story -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger diff --git a/src/pull_requests_parser.py b/src/pull_requests_parser.py index ac177d87..b90400d4 100644 --- a/src/pull_requests_parser.py +++ b/src/pull_requests_parser.py @@ -7,9 +7,10 @@ import pytz import requests +from src.api.baseAPI import IRepositoryAPI from src.constants import EMPTY_FIELD, TIMEDELTA, TIMEZONE from src.git_logger import get_assignee_story -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger from src.repo_dataclasses import PullRequestData diff --git a/src/workflow_runs_parser.py b/src/workflow_runs_parser.py index be69bbf8..25cedae5 100644 --- a/src/workflow_runs_parser.py +++ b/src/workflow_runs_parser.py @@ -2,8 +2,9 @@ from time import sleep from typing import Generator +from src.api.baseAPI import IRepositoryAPI from src.constants import TIMEDELTA -from src.interface_wrapper import IRepositoryAPI, Repository +from src.api.models import Repository from src.utils import logger From ef2f9bff5fdb918f5d4e3e6cb1f75d9d3fccb58d Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 13:55:17 +0300 Subject: [PATCH 3/6] add PoC for Forgejo DB --- main.py | 11 +++- src/api/baseAPI.py | 67 ++++++++++++++++------ src/api/forgejo/ForgejoRepoDB.py | 96 ++++++++++++++++++++++++++++++++ src/api/forgejo/sql_queries.py | 57 +++++++++++++++++++ src/api/utils.py | 7 +++ src/git_logger.py | 29 ++++------ 6 files changed, 229 insertions(+), 38 deletions(-) create mode 100644 src/api/forgejo/ForgejoRepoDB.py create mode 100644 src/api/forgejo/sql_queries.py create mode 100644 src/api/utils.py diff --git a/main.py b/main.py index 1d703264..b2d8b74f 100644 --- a/main.py +++ b/main.py @@ -12,10 +12,18 @@ from src import wikipars from src import workflow_runs_parser from src.utils import parse_time, validate_and_normalize_cell +from src.api.utils import APITypes def parse_args(): parser = argparse.ArgumentParser() + parser.add_argument( + '--source', + choices=list(APITypes), + default=APITypes.github, + required=False, + help='Data source: github API (by default), forgejo API, forgejo db (direct access)' + ) parser.add_argument("--invites", help="print pending invites", action="store_true") parser.add_argument("-c", "--commits", help="log commits", action="store_true") parser.add_argument( @@ -51,6 +59,7 @@ def parse_args(): '--base_url', type=str, required=False, + default="https://git.moevm.pro/api/v1", help='Base URL for Forgejo instance (if using Forgejo)', ) @@ -212,7 +221,7 @@ def main(): repositories = git_logger.get_repos_from_file(args.list) try: - clients = git_logger.Clients(tokens, args.base_url) + clients = git_logger.Clients(api_type=args.source, auth_data=dict(tokens=tokens, base_url=args.base_url)) binded_repos = git_logger.get_next_binded_repo(clients, repositories) except Exception as e: print(f"Failed to initialize any clients: {e}") diff --git a/src/api/baseAPI.py b/src/api/baseAPI.py index 0b8145c8..78657a0c 100644 --- a/src/api/baseAPI.py +++ b/src/api/baseAPI.py @@ -3,9 +3,24 @@ from github import Auth, Github from pyforgejo import PyforgejoApi -from src.api.models import (Branch, Comment, Commit, Contributor, Invite, - Issue, PullRequest, Repository, User, WikiPage, - WorkflowRun) +from src.api.models import ( + Branch, + Comment, + Commit, + Contributor, + Invite, + Issue, + PullRequest, + Repository, + User, + WikiPage, + WorkflowRun, +) + +import logging +import traceback + +from src.api.utils import APITypes # Интерфейс API @@ -80,23 +95,39 @@ def get_base_url(self) -> str: class RepositoryFactory: @staticmethod - def create_api(token: str, base_url: str | None = None) -> IRepositoryAPI: + def create_api(api_type: APITypes, auth_data: dict) -> IRepositoryAPI: from src.api.forgejo.ForgejoRepoAPI import ForgejoRepoAPI from src.api.github.GitHubRepoAPI import GitHubRepoAPI errors = [] - try: - client = GitHubRepoAPI(Github(auth=Auth.Token(token))) - if client.client: - return client - except Exception as e: - errors.append(f"GitHub login failed: {e}") - - if base_url: - try: - return ForgejoRepoAPI(PyforgejoApi(api_key=token, base_url=base_url)) - except Exception as e: - errors.append(f"Forgejo login failed: {e}") - - raise Exception(" / ".join(errors)) + match api_type: + case APITypes.github: + try: + token = auth_data.get("token") + client = GitHubRepoAPI(Github(auth=Auth.Token(token))) + if client.client: + return client + except Exception as e: + errors.append( + f"GitHub login failed: {e}. Token: {str(token)[:4]}..." + ) + case APITypes.forgejo | auth_data.get("base_url"): # legacy code for backward compability + try: + base_url = auth_data.get("base_url") + token = auth_data.get("token") + return ForgejoRepoAPI( + PyforgejoApi(api_key=token, base_url=base_url) + ) + except Exception as e: + errors.append( + f"Forgejo login failed: {e}. Base url: {base_url}. Token: {str(token)[:4]}..." + ) + case APITypes.forgejo_db: + ... + + if errors: + logging.error(" / ".join(errors)) + logging.error("\n".join(traceback.format_stack())) + + return None diff --git a/src/api/forgejo/ForgejoRepoDB.py b/src/api/forgejo/ForgejoRepoDB.py new file mode 100644 index 00000000..b4d1b8f2 --- /dev/null +++ b/src/api/forgejo/ForgejoRepoDB.py @@ -0,0 +1,96 @@ +import base64 +import logging + +import psycopg2 + + +from src.api.baseAPI import IRepositoryAPI +from src.api.models import ( + Branch, + Comment, + Commit, + Contributor, + Issue, + PullRequest, + Repository, + User, + WikiPage, +) +from src.utils import log_exceptions +from src.api.forgejo.sql_queries import GET_FULL_PR_INFO + + +class ForgejoDBClient: + def __init__(self, connect_info: dict): + self.connection_params = dict( + host=connect_info.get("FORGEJO_DB_HOST"), + port=int(connect_info.get("FORGEJO_DB_PORT", 5432)), + dbname=connect_info.get("FORGEJO_DB_NAME"), + user=connect_info.get("FORGEJO_DB_USER"), + password=connect_info.get("FORGEJO_DB_PASSWORD"), + ) + self.connection = psycopg2.connect(**self.connection_params) + + def __enter__(self): + self.connection = psycopg2.connect(**self.connection_params) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.connection and not self.connection.closed: + self.connection.close() + return False + + def get_all_pull_requests(self): + with self.connection.cursor() as cur: + cur.execute(GET_FULL_PR_INFO, ([], )) + columns = [d[0] for d in cur.description] + rows = [dict(zip(columns, r)) for r in cur.fetchall()] + + return [self._row_to_pr(row) for row in rows] + + @staticmethod + def _row_to_pr(row: dict) -> PullRequest: + author = User( + name=row["creator_name"], + login=row["creator_login"], + email=row["creator_email"], + ) + + merged_by = None + if row["merged"] and row["merger_login"]: + merged_by = User( + name=row["merger_name"], + login=row["merger_login"], + email=row["merger_email"], + ) + + return PullRequest( + _id=row["number"], # per-repo номер PR (GitHub API: p.number) + title=row["title"], + author=author, + state=row["state"], # 'open' / 'closed' + created_at=row["created_at"], + head_label=row["source_branch"], + base_label=row["target_branch"], + head_ref=row["source_branch"], + base_ref=row["target_branch"], + merged_by=merged_by, + merged=bool(row["merged"]), + files=[], + issue_url=None, + labels=row["labels"].split(", ") if row["labels"] else [], + milestone=row["milestone"], + comments=row["num_comments"] or 0, + review_comments=0, + ) + +class ForgejoRepoDB(IRepositoryAPI): + def __init__(self, client: ForgejoDBClient): + self.client = client + + @log_exceptions( + default_return=[], message="Failed to get all pull requests from Forgejo DB" + ) + def get_pull_requests(self, repo: Repository) -> list[PullRequest]: + logging.info(f"{self} doesn't support per-repo - it returns all PR in DB") + return self.client.get_all_pull_requests() diff --git a/src/api/forgejo/sql_queries.py b/src/api/forgejo/sql_queries.py new file mode 100644 index 00000000..a3807c2b --- /dev/null +++ b/src/api/forgejo/sql_queries.py @@ -0,0 +1,57 @@ +GET_FULL_PR_INFO = """ +SELECT + r.owner_name || '/' || r.name AS repository_name, + i.name AS title, + i.index AS number, + i.num_comments AS num_comments, + CASE WHEN i.is_closed THEN 'closed' ELSE 'open' END AS state, + pr.base_branch AS commit_into, + pr.head_branch AS commit_from, + to_timestamp(i.created_unix) AS created_at, + creator.name AS creator_name, + creator.lower_name AS creator_login, + creator.email AS creator_email, + NULL::text AS changed_files, + (SELECT string_agg(c.content, E'\\n---\\n' ORDER BY c.created_unix) + FROM comment c WHERE c.issue_id = i.id) AS comment_body, + (SELECT string_agg(to_timestamp(c.created_unix)::text, ', ' ORDER BY c.created_unix) + FROM comment c WHERE c.issue_id = i.id) AS comment_created_at, + (SELECT string_agg(cu.name, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id) AS comment_author_name, + (SELECT string_agg(cu.lower_name, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id) AS comment_author_login, + (SELECT string_agg(cu.email, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id) AS comment_author_email, + merger.name AS merger_name, + merger.lower_name AS merger_login, + merger.email AS merger_email, + pr.head_branch AS source_branch, + pr.base_branch AS target_branch, + (SELECT string_agg(a.name, ', ' ORDER BY a.name) + FROM issue_assignees ia JOIN "user" a ON ia.assignee_id = a.id + WHERE ia.issue_id = i.id) AS assignee_story, + (SELECT string_agg(related_id::text, ', ' ORDER BY related_id) + FROM ( + SELECT dependency_id AS related_id + FROM issue_dependency WHERE issue_id = i.id + UNION + SELECT issue_id AS related_id + FROM issue_dependency WHERE dependency_id = i.id + ) AS rel) AS related_issues, + (SELECT string_agg(l.name, ', ' ORDER BY l.name) + FROM issue_label il JOIN label l ON il.label_id = l.id + WHERE il.issue_id = i.id) AS labels, + m.name AS milestone, + pr.has_merged AS merged +FROM pull_request pr +INNER JOIN issue i ON pr.issue_id = i.id +INNER JOIN repository r ON pr.base_repo_id = r.id +INNER JOIN "user" creator ON i.poster_id = creator.id +LEFT JOIN "user" merger ON pr.merger_id = merger.id +LEFT JOIN milestone m ON i.milestone_id = m.id +WHERE r.owner_name || '/' || r.name = ANY(%s) +ORDER BY i.created_unix DESC; +""" \ No newline at end of file diff --git a/src/api/utils.py b/src/api/utils.py new file mode 100644 index 00000000..3277c48e --- /dev/null +++ b/src/api/utils.py @@ -0,0 +1,7 @@ +from enum import StrEnum + + +class APITypes(StrEnum): + forgejo = "forgejo" + forgejo_db = "forgejo_db" + github = "github" diff --git a/src/git_logger.py b/src/git_logger.py index 98d6888c..acbf4cfe 100644 --- a/src/git_logger.py +++ b/src/git_logger.py @@ -10,22 +10,13 @@ from src.api.baseAPI import ( RepositoryFactory ) +from src.api.utils import APITypes from src.constants import ( EMPTY_FIELD, TIMEDELTA, ) -def login(token, base_url): - try: - client = RepositoryFactory.create_api(token, base_url) - return client - except Exception as e: - logging.error(e) - logging.error(traceback.format_exc()) - return None - - def get_tokens_from_file(tokens_path: str) -> list[str]: with open(tokens_path, 'r') as file: tokens = [token for token in file.read().split('\n') if token] @@ -46,20 +37,20 @@ def get_repos_from_file(repos_path: str) -> list[str]: class Clients: - def __init__(self, tokens: list[str], base_url: str | None = None): + def __init__(self, api_type: APITypes, auth_data: list[dict]): self.clients = [] self.token_map = {} - for token in tokens: - client = login(token, base_url) - if client: - self.clients.append(client) - self.token_map[client] = token + if api_type != APITypes.forgejo_db: + + for auth in auth_data: + client, token = RepositoryFactory.create_api(api_type, auth) + if client: + self.clients.append(client) + self.token_map[client] = token if not self.clients: - if base_url: - raise Exception("No valid tokens for either GitHub or Forgejo") - raise Exception("Make sure that base_url is provided") + raise Exception(f"No clients after login: type={api_type}") def _get_next_client(self) -> tuple[IRepositoryAPI, str]: client = None From 97fc39fb32d27e3f6ab6080ee8ad3b8475784aed Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 15:03:34 +0300 Subject: [PATCH 4/6] add forgejodb exporter for PR --- .dockerignore | 3 +- main.py | 95 ++++++++++++---------- requirements.txt | 9 ++- src/api/baseAPI.py | 25 ++++-- src/api/forgejo/ForgejoRepoDB.py | 133 +++++++++++++++++++++++++------ src/api/forgejo/sql_queries.py | 109 +++++++++++++++++++------ src/git_logger.py | 12 ++- src/pull_requests_parser.py | 2 +- 8 files changed, 272 insertions(+), 116 deletions(-) diff --git a/.dockerignore b/.dockerignore index fd9abfc9..1d219f00 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,4 +2,5 @@ .git forgejo Industrial_Nikolaev_AA_9381 -reports \ No newline at end of file +reports +output \ No newline at end of file diff --git a/main.py b/main.py index b2d8b74f..70f1b447 100644 --- a/main.py +++ b/main.py @@ -18,11 +18,11 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( - '--source', + "--source", choices=list(APITypes), default=APITypes.github, required=False, - help='Data source: github API (by default), forgejo API, forgejo db (direct access)' + help="Data source: github API (by default), forgejo API, forgejo db (direct access)", ) parser.add_argument("--invites", help="print pending invites", action="store_true") parser.add_argument("-c", "--commits", help="log commits", action="store_true") @@ -30,7 +30,9 @@ def parse_args(): "-p", "--pull_requests", help="log pull requests", action="store_true" ) parser.add_argument( - "--graphql", help="use graphql for requesting data (work only with --pull_requests) ", action="store_true" + "--graphql", + help="use graphql for requesting data (work only with --pull_requests) ", + action="store_true", ) parser.add_argument("-i", "--issues", help="log issues", action="store_true") parser.add_argument("-w", "--wikis", help="log wikis", action="store_true") @@ -48,91 +50,91 @@ def parse_args(): action="store_true", ) parser.add_argument( - '--start_cell', + "--start_cell", type=str, required=False, help='Starting cell for Google Sheets export (e.g., "A1", "B3")', - default="A1" + default="A1", ) parser.add_argument( - '--base_url', + "--base_url", type=str, required=False, default="https://git.moevm.pro/api/v1", - help='Base URL for Forgejo instance (if using Forgejo)', + help="Base URL for Forgejo instance (if using Forgejo)", ) token = parser.add_mutually_exclusive_group(required=True) - token.add_argument('-t', '--token', type=str, help='account access token') - token.add_argument('--tokens', type=str, help='path to your tokens') + token.add_argument("-t", "--token", type=str, help="account access token") + token.add_argument("--tokens", type=str, help="path to your tokens") parser.add_argument( - '-l', - '--list', + "-l", + "--list", type=str, required=True, help=( - 'Path to the file containing the list of repositories. ' - 'Repositories should be separated by a line break. ' - 'Names must be in the format / ' + "Path to the file containing the list of repositories. " + "Repositories should be separated by a line break. " + "Names must be in the format / " ), ) parser.add_argument( "--download_repos", type=str, help="path to downloaded repositories", - default='./', + default="./", ) - parser.add_argument('-o', '--out', type=str, required=True, help='output filename') + parser.add_argument("-o", "--out", type=str, required=True, help="output filename") parser.add_argument( "--pr_comments", help="log comments for PR", action="store_true" ) parser.add_argument( - '-s', - '--start', + "-s", + "--start", type=str, required=False, - help='start time', - default='2000/01/01-00:00:00', + help="start time", + default="2000/01/01-00:00:00", ) parser.add_argument( - '-f', - '--finish', + "-f", + "--finish", type=str, required=False, - help='finish time', - default='2400/01/01-00:00:00', + help="finish time", + default="2400/01/01-00:00:00", ) parser.add_argument( - '-b', - '--branch', + "-b", + "--branch", type=str, required=False, help=( - 'branch to select commits, ' + "branch to select commits, " 'by default use "default" repository branch, ' 'use "all" to get all commits from all branches', ), default=None, ) parser.add_argument( - '--google_token', + "--google_token", type=str, required=False, - help='Specify path to google token file', + help="Specify path to google token file", ) parser.add_argument( - '--table_id', + "--table_id", type=str, required=False, - help='Specify Google sheet document id (can find in url)', + help="Specify Google sheet document id (can find in url)", ) parser.add_argument( - '--sheet_name', + "--sheet_name", type=str, required=False, - help='Specify title for a sheet in a document in which data will be printed', + help="Specify title for a sheet in a document in which data will be printed", ) parser.add_argument( "--clear_sheet", @@ -144,18 +146,18 @@ def parse_args(): if args.export_google_sheets: for action in parser._actions: - if action.dest == 'google_token': + if action.dest == "google_token": action.required = True - if action.dest == 'table_id': + if action.dest == "table_id": action.required = True - if action.dest == 'sheet_name': + if action.dest == "sheet_name": action.required = True return parser.parse_args() def run(args, binded_repos, repos_for_wiki=None): - start = parse_time(args.start.split('-')) - finish = parse_time(args.finish.split('-')) + start = parse_time(args.start.split("-")) + finish = parse_time(args.finish.split("-")) if args.commits: commits_parser.log_commits( @@ -164,8 +166,7 @@ def run(args, binded_repos, repos_for_wiki=None): if args.pull_requests: if args.graphql: pull_requests_parser.log_pull_requests_by_graphql( - binded_repos=binded_repos, - csv_name=args.out + binded_repos=binded_repos, csv_name=args.out ) else: pull_requests_parser.log_pull_requests( @@ -178,7 +179,12 @@ def run(args, binded_repos, repos_for_wiki=None): ) if args.issues: issues_parser.log_issues( - binded_repos, args.out, start, finish, args.forks_include, args.base_url, + binded_repos, + args.out, + start, + finish, + args.forks_include, + args.base_url, ) if args.invites: invites_parser.log_invitations( @@ -221,7 +227,10 @@ def main(): repositories = git_logger.get_repos_from_file(args.list) try: - clients = git_logger.Clients(api_type=args.source, auth_data=dict(tokens=tokens, base_url=args.base_url)) + clients = git_logger.Clients( + api_type=args.source, + auth_data=[dict(token=token, base_url=args.base_url) for token in tokens], + ) binded_repos = git_logger.get_next_binded_repo(clients, repositories) except Exception as e: print(f"Failed to initialize any clients: {e}") @@ -231,5 +240,5 @@ def main(): run(args, binded_repos, repositories) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/requirements.txt b/requirements.txt index 800f5b1f..a2257550 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ GitPython==3.1.41 +isodate==0.7.2 +pandas==2.2.3 +psycopg2-binary==2.9.13 +pyforgejo==2.0.7 PyGithub==2.4.0 pygsheets==2.0.6 -pandas==2.2.3 pytz==2024.2 requests==2.32.3 -pyforgejo==2.0.7 -isodate==0.7.2 -unittest-parametrize==1.6.0 +unittest-parametrize==1.6.0 \ No newline at end of file diff --git a/src/api/baseAPI.py b/src/api/baseAPI.py index 78657a0c..24904e12 100644 --- a/src/api/baseAPI.py +++ b/src/api/baseAPI.py @@ -97,37 +97,46 @@ class RepositoryFactory: @staticmethod def create_api(api_type: APITypes, auth_data: dict) -> IRepositoryAPI: from src.api.forgejo.ForgejoRepoAPI import ForgejoRepoAPI + from src.api.forgejo.ForgejoRepoDB import ForgejoDBClient, ForgejoRepoDB from src.api.github.GitHubRepoAPI import GitHubRepoAPI errors = [] + base_url = auth_data.get("base_url") + token = auth_data.get("token") + if auth_data.get("base_url") and api_type != APITypes.forgejo_db: + api_type = APITypes.forgejo # legacy code for backward compability + match api_type: case APITypes.github: try: - token = auth_data.get("token") client = GitHubRepoAPI(Github(auth=Auth.Token(token))) if client.client: - return client + return client, token except Exception as e: errors.append( f"GitHub login failed: {e}. Token: {str(token)[:4]}..." ) - case APITypes.forgejo | auth_data.get("base_url"): # legacy code for backward compability + case APITypes.forgejo: try: - base_url = auth_data.get("base_url") - token = auth_data.get("token") return ForgejoRepoAPI( PyforgejoApi(api_key=token, base_url=base_url) - ) + ), token except Exception as e: errors.append( f"Forgejo login failed: {e}. Base url: {base_url}. Token: {str(token)[:4]}..." ) case APITypes.forgejo_db: - ... + try: + token = auth_data.get("token") + return ForgejoRepoDB(ForgejoDBClient({'base_url': base_url, 'FORGEJO_DB_PASSWORD': token})), None + except Exception as e: + errors.append( + f"Forgejo DB login failed: {e}. Base url: {base_url}. Token: {str(token)[:4]}..." + ) if errors: logging.error(" / ".join(errors)) logging.error("\n".join(traceback.format_stack())) - return None + return None, None diff --git a/src/api/forgejo/ForgejoRepoDB.py b/src/api/forgejo/ForgejoRepoDB.py index b4d1b8f2..3cd607df 100644 --- a/src/api/forgejo/ForgejoRepoDB.py +++ b/src/api/forgejo/ForgejoRepoDB.py @@ -17,19 +17,20 @@ WikiPage, ) from src.utils import log_exceptions -from src.api.forgejo.sql_queries import GET_FULL_PR_INFO +from src.api.forgejo.sql_queries import GET_FULL_PR_INFO, GET_USER_BY_ID, GET_REPO class ForgejoDBClient: def __init__(self, connect_info: dict): self.connection_params = dict( - host=connect_info.get("FORGEJO_DB_HOST"), + host=connect_info.get("FORGEJO_DB_HOST", 'db'), port=int(connect_info.get("FORGEJO_DB_PORT", 5432)), - dbname=connect_info.get("FORGEJO_DB_NAME"), - user=connect_info.get("FORGEJO_DB_USER"), - password=connect_info.get("FORGEJO_DB_PASSWORD"), + dbname=connect_info.get("FORGEJO_DB_NAME", 'forgejo'), + user=connect_info.get("FORGEJO_DB_USER", "forgejo"), + password=connect_info.get("FORGEJO_DB_PASSWORD", "moevmforgejopassword"), ) - self.connection = psycopg2.connect(**self.connection_params) + self.connection = None + self.base_url = connect_info.get("base_url") def __enter__(self): self.connection = psycopg2.connect(**self.connection_params) @@ -41,34 +42,45 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False def get_all_pull_requests(self): - with self.connection.cursor() as cur: - cur.execute(GET_FULL_PR_INFO, ([], )) - columns = [d[0] for d in cur.description] - rows = [dict(zip(columns, r)) for r in cur.fetchall()] - + with self: + with self.connection.cursor() as cur: + cur.execute(GET_FULL_PR_INFO) + columns = [d[0] for d in cur.description] + rows = [dict(zip(columns, r)) for r in cur.fetchall()] + logging.error(f"{rows}") return [self._row_to_pr(row) for row in rows] @staticmethod - def _row_to_pr(row: dict) -> PullRequest: - author = User( - name=row["creator_name"], - login=row["creator_login"], - email=row["creator_email"], + def _user_from_row(row: dict, prefix: str, base_url: str = "") -> User | None: + uid = row.get(f"{prefix}_id") + if not uid: + return None + + login = row.get(f"{prefix}_login") or "" + return User( + _id=uid, + login=login, + username=row.get(f"{prefix}_full_name") or row.get(f"{prefix}_name") or "No name", + email=row.get(f"{prefix}_email") or "", + html_url=f"{base_url.rstrip('/')}/{login}" if base_url and login else "", + node_id=str(uid), + type={0: "User", 1: "Organization"}.get(row.get(f"{prefix}_type") or 0, ""), + bio=row.get(f"{prefix}_bio") or "", + site_admin=bool(row.get(f"{prefix}_is_admin")), ) - merged_by = None - if row["merged"] and row["merger_login"]: - merged_by = User( - name=row["merger_name"], - login=row["merger_login"], - email=row["merger_email"], - ) + def _row_to_pr(self, row: dict) -> PullRequest: + author = self._user_from_row(row, "creator", self.base_url) + merged_by = ( + self._user_from_row(row, "merger", self.base_url) + if row["merged"] else None + ) return PullRequest( - _id=row["number"], # per-repo номер PR (GitHub API: p.number) + _id=row["number"], title=row["title"], author=author, - state=row["state"], # 'open' / 'closed' + state=row["state"], created_at=row["created_at"], head_label=row["source_branch"], base_label=row["target_branch"], @@ -84,13 +96,82 @@ def _row_to_pr(row: dict) -> PullRequest: review_comments=0, ) -class ForgejoRepoDB(IRepositoryAPI): +class ForgejoRepoDB(): + _USER_TYPES = {0: "User", 1: "Organization", 2: "Bot"} + def __init__(self, client: ForgejoDBClient): self.client = client + def get_user_data(self, row: dict) -> User: + """ + row — dict w/keys: id, name, lower_name, full_name, + email, description, is_admin, type. + """ + login = row.get("lower_name") or row.get("name") or "" + return User( + login=login, + username=row.get("full_name") or row.get("name") or "No name", + email=row.get("email") or "", + html_url=f"{self.client.base_url}/{login}" if self.client.base_url else None, + node_id=row.get("id"), + type=self._USER_TYPES.get(row.get("type") or 0, ""), + bio=row.get("description") or "", + site_admin=bool(row.get("is_admin")), + _id=row.get("id"), + ) + + def get_user_by_id(self, user_id: int) -> User | None: + with self.client: + with self.client.connection.cursor() as cur: + cur.execute(GET_USER_BY_ID, (user_id,)) + row = cur.fetchone() + if not row: + return None + cols = [d[0] for d in cur.description] + return self.get_user_data(dict(zip(cols, row))) + + @log_exceptions(default_return=None, message="Failed to get repository from Forgejo DB") + def get_repository(self, id: str) -> Repository | None: + logging.info(f"{self} doesn't support get_repository (it parses all PR in DB), so returned Repo is Mock") + with self.client: + with self.client.connection.cursor() as cur: + cur.execute(GET_REPO, (id,)) + row = cur.fetchone() + if not row: + logging.error(f"Repository {id} not found in Forgejo DB.") + return None + cols = [d[0] for d in cur.description] + row = dict(zip(cols, row)) + + owner = self.get_user_data({ + "id": row["owner_id"], + "name": row["owner_name_user"], + "lower_name": row["owner_login"], + "full_name": row["owner_full_name"], + "email": row["owner_email"], + "description": row["owner_description"], + "is_admin": row["owner_is_admin"], + "type": row["owner_type"], + }) + + return Repository( + _id=f"{row['owner_name']}/{row['name']}", + name=row["name"], + url=f"{self.client.base_url}/{row['owner_name']}/{row['name']}" if self.client.base_url else None, + default_branch=Branch(name=row["default_branch"], last_commit=None), + owner=owner, + ) + @log_exceptions( default_return=[], message="Failed to get all pull requests from Forgejo DB" ) def get_pull_requests(self, repo: Repository) -> list[PullRequest]: logging.info(f"{self} doesn't support per-repo - it returns all PR in DB") return self.client.get_all_pull_requests() + + def get_rate_limiting(self) -> tuple[int, int]: + import sys + return sys.maxsize, sys.maxsize + + def get_base_url(self): + return self.client.base_url \ No newline at end of file diff --git a/src/api/forgejo/sql_queries.py b/src/api/forgejo/sql_queries.py index a3807c2b..d2f4574e 100644 --- a/src/api/forgejo/sql_queries.py +++ b/src/api/forgejo/sql_queries.py @@ -1,38 +1,53 @@ GET_FULL_PR_INFO = """ SELECT + -- Репозиторий r.owner_name || '/' || r.name AS repository_name, - i.name AS title, + + -- Основные поля PR (для PullRequest) i.index AS number, - i.num_comments AS num_comments, + i.name AS title, CASE WHEN i.is_closed THEN 'closed' ELSE 'open' END AS state, - pr.base_branch AS commit_into, - pr.head_branch AS commit_from, to_timestamp(i.created_unix) AS created_at, + pr.head_branch AS source_branch, + pr.base_branch AS target_branch, + pr.head_branch AS head_ref, + pr.base_branch AS base_ref, + pr.has_merged AS merged, + i.num_comments AS num_comments, + 0 AS review_comments, + m.name AS milestone, + + -- Автор PR (для User) + creator.id AS creator_id, creator.name AS creator_name, creator.lower_name AS creator_login, + creator.full_name AS creator_full_name, creator.email AS creator_email, - NULL::text AS changed_files, - (SELECT string_agg(c.content, E'\\n---\\n' ORDER BY c.created_unix) - FROM comment c WHERE c.issue_id = i.id) AS comment_body, - (SELECT string_agg(to_timestamp(c.created_unix)::text, ', ' ORDER BY c.created_unix) - FROM comment c WHERE c.issue_id = i.id) AS comment_created_at, - (SELECT string_agg(cu.name, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id) AS comment_author_name, - (SELECT string_agg(cu.lower_name, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id) AS comment_author_login, - (SELECT string_agg(cu.email, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id) AS comment_author_email, + creator.description AS creator_bio, + creator.is_admin AS creator_is_admin, + creator.type AS creator_type, + + -- Мержер (для User) + merger.id AS merger_id, merger.name AS merger_name, merger.lower_name AS merger_login, + merger.full_name AS merger_full_name, merger.email AS merger_email, - pr.head_branch AS source_branch, - pr.base_branch AS target_branch, + merger.description AS merger_bio, + merger.is_admin AS merger_is_admin, + merger.type AS merger_type, + + -- Лейблы + (SELECT string_agg(l.name, ', ' ORDER BY l.name) + FROM issue_label il JOIN label l ON il.label_id = l.id + WHERE il.issue_id = i.id) AS labels, + + -- Назначенные исполнители (SELECT string_agg(a.name, ', ' ORDER BY a.name) FROM issue_assignees ia JOIN "user" a ON ia.assignee_id = a.id WHERE ia.issue_id = i.id) AS assignee_story, + + -- Связанные issues (SELECT string_agg(related_id::text, ', ' ORDER BY related_id) FROM ( SELECT dependency_id AS related_id @@ -41,17 +56,59 @@ SELECT issue_id AS related_id FROM issue_dependency WHERE dependency_id = i.id ) AS rel) AS related_issues, - (SELECT string_agg(l.name, ', ' ORDER BY l.name) - FROM issue_label il JOIN label l ON il.label_id = l.id - WHERE il.issue_id = i.id) AS labels, - m.name AS milestone, - pr.has_merged AS merged + + -- Комментарии (только пользовательские, type = 0) + (SELECT string_agg(c.content, E'\n---\n' ORDER BY c.created_unix) + FROM comment c + WHERE c.issue_id = i.id AND c.type = 0) AS comment_body, + (SELECT string_agg(to_timestamp(c.created_unix)::text, ', ' ORDER BY c.created_unix) + FROM comment c + WHERE c.issue_id = i.id AND c.type = 0) AS comment_created_at, + (SELECT string_agg(cu.name, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_name, + (SELECT string_agg(cu.lower_name, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_login, + (SELECT string_agg(cu.email, ', ' ORDER BY c.created_unix) + FROM comment c JOIN "user" cu ON c.poster_id = cu.id + WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_email, + + -- Изменённые файлы (через БД недоступны) + NULL::text AS changed_files + FROM pull_request pr INNER JOIN issue i ON pr.issue_id = i.id INNER JOIN repository r ON pr.base_repo_id = r.id INNER JOIN "user" creator ON i.poster_id = creator.id LEFT JOIN "user" merger ON pr.merger_id = merger.id LEFT JOIN milestone m ON i.milestone_id = m.id -WHERE r.owner_name || '/' || r.name = ANY(%s) ORDER BY i.created_unix DESC; +""" + +GET_USER_BY_ID = """ +SELECT id, name, lower_name, full_name, email, description, is_admin, type + FROM "user" WHERE id = %s; +""" + +GET_REPO = """ +SELECT + r.id, + r.owner_name, + r.name, + r.lower_name, + r.description, + r.default_branch, + r.owner_id, + u.id AS owner_id, + u.name AS owner_name_user, + u.lower_name AS owner_login, + u.full_name AS owner_full_name, + u.email AS owner_email, + u.description AS owner_description, + u.is_admin AS owner_is_admin, + u.type AS owner_type +FROM repository r +LEFT JOIN "user" u ON r.owner_id = u.id +WHERE r.owner_name || '/' || r.name = %s; """ \ No newline at end of file diff --git a/src/git_logger.py b/src/git_logger.py index acbf4cfe..7a0ff886 100644 --- a/src/git_logger.py +++ b/src/git_logger.py @@ -41,13 +41,11 @@ def __init__(self, api_type: APITypes, auth_data: list[dict]): self.clients = [] self.token_map = {} - if api_type != APITypes.forgejo_db: - - for auth in auth_data: - client, token = RepositoryFactory.create_api(api_type, auth) - if client: - self.clients.append(client) - self.token_map[client] = token + for auth in auth_data: + client, token = RepositoryFactory.create_api(api_type, auth) + if client: + self.clients.append(client) + self.token_map[client] = token if not self.clients: raise Exception(f"No clients after login: type={api_type}") diff --git a/src/pull_requests_parser.py b/src/pull_requests_parser.py index b90400d4..53033734 100644 --- a/src/pull_requests_parser.py +++ b/src/pull_requests_parser.py @@ -123,7 +123,7 @@ def get_info(obj, attr): merged=pull.merged, source_branch=pull.head_ref, target_branch=pull.base_ref, - assignee_story=get_assignee_story(pull, client, token, repository), + assignee_story=None, #get_assignee_story(pull, client, token, repository), related_issues=( get_related_issues(pull._id, repository.owner, repository.name, token) if pull.issue_url is not None From 6c77ea68d641c7b6d83f49ffd487a1e77800dab3 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 15:14:25 +0300 Subject: [PATCH 5/6] Rm default value for base_url --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 70f1b447..3fdc821c 100644 --- a/main.py +++ b/main.py @@ -61,7 +61,6 @@ def parse_args(): "--base_url", type=str, required=False, - default="https://git.moevm.pro/api/v1", help="Base URL for Forgejo instance (if using Forgejo)", ) From a0a940f68b47b5b8ad28ffcdaada4b7c82cd3187 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 26 Sep 2026 20:38:42 +0300 Subject: [PATCH 6/6] update reponame getter for PR export --- README.md | 12 ++++++ main.py | 41 +++++++++++++++--- src/api/baseAPI.py | 6 ++- src/api/forgejo/ForgejoRepoDB.py | 73 +++++++++++++++++++------------- src/api/forgejo/sql_queries.py | 51 +++++----------------- src/api/models.py | 1 + src/commits_parser.py | 2 +- src/git_logger.py | 3 +- src/invites_parser.py | 2 +- src/issues_parser.py | 2 +- src/pull_requests_parser.py | 7 ++- src/utils.py | 66 ++++++++++++++++++++++------- src/wikipars.py | 3 +- 13 files changed, 167 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index f173462c..2bd4c31f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,11 @@ docker run -v $(pwd)/output:/app/output checking_repo [--invites] [--commites] [ 6. Нажимаем Generate token, сохраняем код токена и пользуемся. ## Запуск приложения: +Без `--debug` выводятся прогресс записи CSV раз в 5 секунд, итоговое число строк, +предупреждения и ошибки. Параметр `--debug` включает подробные логи отдельных записей. +CSV-файл и writer переиспользуются, после каждой строки вызывается `flush()`: +уже записанные строки сохраняются при обрыве процесса, файл закрывается при завершении обработки. + 1. Логирование commits ```commandline python3 main.py [-c, --commits] (-t token (github токен вместо token) | --tokens tokens (путь до файла с токенами вместо tokens)) [-l, --list] list (list - строка пути к txt файлу со списком репозиториев) [-o, --out] out (out - название csv файла, в который будут помещены все логи) [-b, --branch] branch (branch - название конкретной ветки, откуда брать коммиты или all - логгировать все коммиты изо всех веток) @@ -41,6 +46,13 @@ python3 main.py [-i, --issues] (-t token (github токен вместо token) ```commandline python3 main.py [-p, --pull_requests] (-t token (github токен вместо token) | --tokens tokens (путь до файла с токенами вместо tokens)) [-l, --list] list (list - строка пути к txt файлу со списком репозиториев) [-o, --out] out (out - название csv файла, в который будут помещены все логи) [--pr_comments] (если установлен - также выгружаются комментарии к PR) ``` +Для прямой выгрузки PR из БД Forgejo (`--source forgejo_db`) можно добавить +`--forgejo-db-organization my-org --forgejo-db-repo-name-contains 2026 2027`: +выбираются PR репозиториев владельца `my-org`, имя которых содержит `2026` +**или** `2027`. Оба фильтра необязательны, применяются в SQL без учёта регистра; +организация задаётся точным логином, подстроки имени трактуются буквально. +Без фильтров выгружаются все PR инстанса; `--list` выборку PR не ограничивает. + 4. Логирование непринятых приглашений в репо ```commandline python3 main.py --invites (-t token (github токен вместо token) | --tokens tokens (путь до файла с токенами вместо tokens)) [-l, --list] list (list - строка пути к txt файлу со списком репозиториев) [-o, --out] out (out - название csv файла, в который будут помещены все логи) diff --git a/main.py b/main.py index 3fdc821c..118990b7 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ import argparse +import logging import traceback import sys @@ -11,12 +12,13 @@ from src import pull_requests_parser from src import wikipars from src import workflow_runs_parser -from src.utils import parse_time, validate_and_normalize_cell +from src.utils import logger, parse_time, validate_and_normalize_cell from src.api.utils import APITypes def parse_args(): parser = argparse.ArgumentParser() + parser.add_argument("--debug", action="store_true", help="Enable detailed console logs") parser.add_argument( "--source", choices=list(APITypes), @@ -25,6 +27,18 @@ def parse_args(): help="Data source: github API (by default), forgejo API, forgejo db (direct access)", ) parser.add_argument("--invites", help="print pending invites", action="store_true") + parser.add_argument( + "--forgejo-db-organization", + help="Forgejo DB only: exact repository owner login (case-insensitive)", + ) + parser.add_argument( + "--forgejo-db-repo-name-contains", + nargs="+", + help=( + "Forgejo DB only: select repositories whose name contains any of " + "these literal substrings (case-insensitive), e.g. 2026 2027" + ), + ) parser.add_argument("-c", "--commits", help="log commits", action="store_true") parser.add_argument( "-p", "--pull_requests", help="log pull requests", action="store_true" @@ -199,6 +213,7 @@ def run(args, binded_repos, repos_for_wiki=None): if args.wikis: wikipars.wikiparser(repos_for_wiki, args.download_repos, args.out) if args.export_google_sheets: + logger.close_csv_files() export_sheets.write_data_to_table( csv_path=args.out, google_token=args.google_token, @@ -211,11 +226,12 @@ def run(args, binded_repos, repos_for_wiki=None): def main(): args = parse_args() + logging.getLogger().setLevel(logging.DEBUG if args.debug else logging.WARNING) try: args.start_cell = validate_and_normalize_cell(args.start_cell) except ValueError as e: - print(f"Error in start_cell argument: {e}") + logging.error("Error in start_cell argument: %s", e) sys.exit(1) if args.token: @@ -228,15 +244,28 @@ def main(): try: clients = git_logger.Clients( api_type=args.source, - auth_data=[dict(token=token, base_url=args.base_url) for token in tokens], + auth_data=[ + dict( + token=token, + base_url=args.base_url, + forgejo_db=dict( + repo_name_contains=args.forgejo_db_repo_name_contains, + organization=args.forgejo_db_organization, + ), + ) + for token in tokens + ], ) binded_repos = git_logger.get_next_binded_repo(clients, repositories) except Exception as e: - print(f"Failed to initialize any clients: {e}") - print(traceback.format_exc()) + logging.error("Failed to initialize any clients: %s", e) + logging.debug(traceback.format_exc()) return - run(args, binded_repos, repositories) + try: + run(args, binded_repos, repositories) + finally: + logger.close_csv_files() if __name__ == "__main__": diff --git a/src/api/baseAPI.py b/src/api/baseAPI.py index 24904e12..2776e06a 100644 --- a/src/api/baseAPI.py +++ b/src/api/baseAPI.py @@ -129,7 +129,11 @@ def create_api(api_type: APITypes, auth_data: dict) -> IRepositoryAPI: case APITypes.forgejo_db: try: token = auth_data.get("token") - return ForgejoRepoDB(ForgejoDBClient({'base_url': base_url, 'FORGEJO_DB_PASSWORD': token})), None + return ForgejoRepoDB(ForgejoDBClient({ + 'base_url': base_url, + 'FORGEJO_DB_PASSWORD': token, + **auth_data.get('forgejo_db', {}), + })), None except Exception as e: errors.append( f"Forgejo DB login failed: {e}. Base url: {base_url}. Token: {str(token)[:4]}..." diff --git a/src/api/forgejo/ForgejoRepoDB.py b/src/api/forgejo/ForgejoRepoDB.py index 3cd607df..f0cb2d9d 100644 --- a/src/api/forgejo/ForgejoRepoDB.py +++ b/src/api/forgejo/ForgejoRepoDB.py @@ -1,4 +1,3 @@ -import base64 import logging import psycopg2 @@ -7,14 +6,9 @@ from src.api.baseAPI import IRepositoryAPI from src.api.models import ( Branch, - Comment, - Commit, - Contributor, - Issue, PullRequest, Repository, User, - WikiPage, ) from src.utils import log_exceptions from src.api.forgejo.sql_queries import GET_FULL_PR_INFO, GET_USER_BY_ID, GET_REPO @@ -23,14 +17,16 @@ class ForgejoDBClient: def __init__(self, connect_info: dict): self.connection_params = dict( - host=connect_info.get("FORGEJO_DB_HOST", 'db'), + host=connect_info.get("FORGEJO_DB_HOST", "db"), port=int(connect_info.get("FORGEJO_DB_PORT", 5432)), - dbname=connect_info.get("FORGEJO_DB_NAME", 'forgejo'), + dbname=connect_info.get("FORGEJO_DB_NAME", "forgejo"), user=connect_info.get("FORGEJO_DB_USER", "forgejo"), - password=connect_info.get("FORGEJO_DB_PASSWORD", "moevmforgejopassword"), + password=connect_info.get("FORGEJO_DB_PASSWORD", ""), ) self.connection = None self.base_url = connect_info.get("base_url") + self.repo_name_contains = connect_info.get("repo_name_contains") or None + self.organization = connect_info.get("organization") def __enter__(self): self.connection = psycopg2.connect(**self.connection_params) @@ -44,10 +40,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): def get_all_pull_requests(self): with self: with self.connection.cursor() as cur: - cur.execute(GET_FULL_PR_INFO) + cur.execute( + GET_FULL_PR_INFO, + dict(repo_name_contains=self.repo_name_contains, + organization=self.organization), + ) columns = [d[0] for d in cur.description] rows = [dict(zip(columns, r)) for r in cur.fetchall()] - logging.error(f"{rows}") return [self._row_to_pr(row) for row in rows] @staticmethod @@ -60,7 +59,9 @@ def _user_from_row(row: dict, prefix: str, base_url: str = "") -> User | None: return User( _id=uid, login=login, - username=row.get(f"{prefix}_full_name") or row.get(f"{prefix}_name") or "No name", + username=row.get(f"{prefix}_full_name") + or row.get(f"{prefix}_name") + or "No name", email=row.get(f"{prefix}_email") or "", html_url=f"{base_url.rstrip('/')}/{login}" if base_url and login else "", node_id=str(uid), @@ -72,11 +73,11 @@ def _user_from_row(row: dict, prefix: str, base_url: str = "") -> User | None: def _row_to_pr(self, row: dict) -> PullRequest: author = self._user_from_row(row, "creator", self.base_url) merged_by = ( - self._user_from_row(row, "merger", self.base_url) - if row["merged"] else None + self._user_from_row(row, "merger", self.base_url) if row["merged"] else None ) return PullRequest( + repository_name=row["repository_name"], _id=row["number"], title=row["title"], author=author, @@ -96,7 +97,8 @@ def _row_to_pr(self, row: dict) -> PullRequest: review_comments=0, ) -class ForgejoRepoDB(): + +class ForgejoRepoDB: # IRepositoryAPI _USER_TYPES = {0: "User", 1: "Organization", 2: "Bot"} def __init__(self, client: ForgejoDBClient): @@ -112,7 +114,9 @@ def get_user_data(self, row: dict) -> User: login=login, username=row.get("full_name") or row.get("name") or "No name", email=row.get("email") or "", - html_url=f"{self.client.base_url}/{login}" if self.client.base_url else None, + html_url=f"{self.client.base_url}/{login}" + if self.client.base_url + else None, node_id=row.get("id"), type=self._USER_TYPES.get(row.get("type") or 0, ""), bio=row.get("description") or "", @@ -130,9 +134,13 @@ def get_user_by_id(self, user_id: int) -> User | None: cols = [d[0] for d in cur.description] return self.get_user_data(dict(zip(cols, row))) - @log_exceptions(default_return=None, message="Failed to get repository from Forgejo DB") + @log_exceptions( + default_return=None, message="Failed to get repository from Forgejo DB" + ) def get_repository(self, id: str) -> Repository | None: - logging.info(f"{self} doesn't support get_repository (it parses all PR in DB), so returned Repo is Mock") + logging.info( + f"{self} doesn't support get_repository (it parses all PR in DB), so returned Repo is Mock" + ) with self.client: with self.client.connection.cursor() as cur: cur.execute(GET_REPO, (id,)) @@ -143,21 +151,25 @@ def get_repository(self, id: str) -> Repository | None: cols = [d[0] for d in cur.description] row = dict(zip(cols, row)) - owner = self.get_user_data({ - "id": row["owner_id"], - "name": row["owner_name_user"], - "lower_name": row["owner_login"], - "full_name": row["owner_full_name"], - "email": row["owner_email"], - "description": row["owner_description"], - "is_admin": row["owner_is_admin"], - "type": row["owner_type"], - }) + owner = self.get_user_data( + { + "id": row["owner_id"], + "name": row["owner_name_user"], + "lower_name": row["owner_login"], + "full_name": row["owner_full_name"], + "email": row["owner_email"], + "description": row["owner_description"], + "is_admin": row["owner_is_admin"], + "type": row["owner_type"], + } + ) return Repository( _id=f"{row['owner_name']}/{row['name']}", name=row["name"], - url=f"{self.client.base_url}/{row['owner_name']}/{row['name']}" if self.client.base_url else None, + url=f"{self.client.base_url}/{row['owner_name']}/{row['name']}" + if self.client.base_url + else None, default_branch=Branch(name=row["default_branch"], last_commit=None), owner=owner, ) @@ -171,7 +183,8 @@ def get_pull_requests(self, repo: Repository) -> list[PullRequest]: def get_rate_limiting(self) -> tuple[int, int]: import sys + return sys.maxsize, sys.maxsize def get_base_url(self): - return self.client.base_url \ No newline at end of file + return self.client.base_url diff --git a/src/api/forgejo/sql_queries.py b/src/api/forgejo/sql_queries.py index d2f4574e..18dcb68e 100644 --- a/src/api/forgejo/sql_queries.py +++ b/src/api/forgejo/sql_queries.py @@ -10,11 +10,8 @@ to_timestamp(i.created_unix) AS created_at, pr.head_branch AS source_branch, pr.base_branch AS target_branch, - pr.head_branch AS head_ref, - pr.base_branch AS base_ref, pr.has_merged AS merged, i.num_comments AS num_comments, - 0 AS review_comments, m.name AS milestone, -- Автор PR (для User) @@ -40,42 +37,7 @@ -- Лейблы (SELECT string_agg(l.name, ', ' ORDER BY l.name) FROM issue_label il JOIN label l ON il.label_id = l.id - WHERE il.issue_id = i.id) AS labels, - - -- Назначенные исполнители - (SELECT string_agg(a.name, ', ' ORDER BY a.name) - FROM issue_assignees ia JOIN "user" a ON ia.assignee_id = a.id - WHERE ia.issue_id = i.id) AS assignee_story, - - -- Связанные issues - (SELECT string_agg(related_id::text, ', ' ORDER BY related_id) - FROM ( - SELECT dependency_id AS related_id - FROM issue_dependency WHERE issue_id = i.id - UNION - SELECT issue_id AS related_id - FROM issue_dependency WHERE dependency_id = i.id - ) AS rel) AS related_issues, - - -- Комментарии (только пользовательские, type = 0) - (SELECT string_agg(c.content, E'\n---\n' ORDER BY c.created_unix) - FROM comment c - WHERE c.issue_id = i.id AND c.type = 0) AS comment_body, - (SELECT string_agg(to_timestamp(c.created_unix)::text, ', ' ORDER BY c.created_unix) - FROM comment c - WHERE c.issue_id = i.id AND c.type = 0) AS comment_created_at, - (SELECT string_agg(cu.name, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_name, - (SELECT string_agg(cu.lower_name, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_login, - (SELECT string_agg(cu.email, ', ' ORDER BY c.created_unix) - FROM comment c JOIN "user" cu ON c.poster_id = cu.id - WHERE c.issue_id = i.id AND c.type = 0) AS comment_author_email, - - -- Изменённые файлы (через БД недоступны) - NULL::text AS changed_files + WHERE il.issue_id = i.id) AS labels FROM pull_request pr INNER JOIN issue i ON pr.issue_id = i.id @@ -83,6 +45,15 @@ INNER JOIN "user" creator ON i.poster_id = creator.id LEFT JOIN "user" merger ON pr.merger_id = merger.id LEFT JOIN milestone m ON i.milestone_id = m.id +WHERE ( + %(repo_name_contains)s::text[] IS NULL + OR EXISTS ( + SELECT 1 + FROM unnest(%(repo_name_contains)s::text[]) AS filter(name_part) + WHERE strpos(lower(r.name), lower(filter.name_part)) > 0 + ) +) +AND (%(organization)s IS NULL OR lower(r.owner_name) = lower(%(organization)s)) ORDER BY i.created_unix DESC; """ @@ -111,4 +82,4 @@ FROM repository r LEFT JOIN "user" u ON r.owner_id = u.id WHERE r.owner_name || '/' || r.name = %s; -""" \ No newline at end of file +""" diff --git a/src/api/models.py b/src/api/models.py index 5d442a29..07022dde 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -88,6 +88,7 @@ class PullRequest: milestone: str comments: int = 0 review_comments: int = 0 + repository_name: str = "" @dataclass diff --git a/src/commits_parser.py b/src/commits_parser.py index 20843215..a4723e50 100644 --- a/src/commits_parser.py +++ b/src/commits_parser.py @@ -39,7 +39,7 @@ def log_repository_commits( branches.append(branch) for branch in branches: - print(f'Processing branch {branch}') + logger.log_to_stdout(f'Processing branch {branch}') commits = client.get_commits(repository) for commit in commits: if ( diff --git a/src/git_logger.py b/src/git_logger.py index 7a0ff886..96d78c00 100644 --- a/src/git_logger.py +++ b/src/git_logger.py @@ -75,8 +75,7 @@ def get_next_binded_repo(clients: Clients, repositories: list[str]): client, token = clients.get_next_client() repo = client.get_repository(repo_name) except Exception as err: - print(f'git_logger.get_next_binded_repo(): error {err}') - print(f'git_logger.get_next_binded_repo(): failed to load repository "{repo_name}"') + logging.error('Failed to load repository %s: %s', repo_name, err) else: yield client, repo, token diff --git a/src/invites_parser.py b/src/invites_parser.py index 503975d4..3e93f031 100644 --- a/src/invites_parser.py +++ b/src/invites_parser.py @@ -41,6 +41,6 @@ def log_invitations( logger.log_to_csv(csv_name, list(info.keys())) for client, repo, token in binded_repos: - print(client, repo, token) + logger.log_to_stdout(str(repo)) logger.log_title(repo.name) log_repository_invitations(client, repo, csv_name) diff --git a/src/issues_parser.py b/src/issues_parser.py index dc199db8..fb1fabb9 100644 --- a/src/issues_parser.py +++ b/src/issues_parser.py @@ -89,7 +89,7 @@ def get_connected_pulls( connected_prs.add(pr.get("html_url")) except requests.exceptions.RequestException as e: - print(f"[Warning] Failed to fetch connected PRs: {str(e)}") + logger.log_warning(f"Failed to fetch connected PRs: {e}") return 'Empty field' return ';'.join(sorted(connected_prs)) if connected_prs else 'Empty field' diff --git a/src/pull_requests_parser.py b/src/pull_requests_parser.py index 53033734..8a31867f 100644 --- a/src/pull_requests_parser.py +++ b/src/pull_requests_parser.py @@ -8,6 +8,7 @@ import requests from src.api.baseAPI import IRepositoryAPI +from src.api.forgejo.ForgejoRepoDB import ForgejoRepoDB from src.constants import EMPTY_FIELD, TIMEDELTA, TIMEZONE from src.git_logger import get_assignee_story from src.api.models import Repository @@ -97,6 +98,7 @@ def nvl(val): def get_info(obj, attr): return EMPTY_FIELD if obj is None else getattr(obj, attr) + is_forgejo_db = isinstance(client, ForgejoRepoDB) pulls = client.get_pull_requests(repository) for pull in pulls: if ( @@ -106,7 +108,7 @@ def get_info(obj, attr): continue pr_data = PullRequestData( - repository_name=repository.name, + repository_name=pull.repository_name or repository.name, title=pull.title, id=pull._id, state=pull.state, @@ -162,7 +164,8 @@ def get_info(obj, attr): logger.log_to_csv(csv_name, list(base_pr_info.keys()), base_pr_info) logger.log_to_stdout(base_pr_info) - sleep(TIMEDELTA) + if not is_forgejo_db: + sleep(TIMEDELTA) def log_pull_requests( diff --git a/src/utils.py b/src/utils.py index 9c349bb0..ff4793af 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,8 +1,10 @@ import csv +import atexit from datetime import datetime from functools import wraps import traceback import re +from time import monotonic import pytz @@ -17,7 +19,22 @@ class logger: - # TODO: отключение вывода в stdout + _csv_files = {} + _csv_progress = {} + _progress_log = logging.getLogger("export.progress") + _progress_log.setLevel(logging.INFO) + + @staticmethod + def close_csv_files(): + try: + for file, _, _ in logger._csv_files.values(): + file.close() + for csv_name, (count, _) in logger._csv_progress.items(): + logger._progress_log.info("CSV %s: total %d rows written", csv_name, count) + finally: + logger._csv_files.clear() + logger._csv_progress.clear() + @staticmethod def log_title(title: str, title_len: int = TITLE_LEN): final_len = max( @@ -25,31 +42,45 @@ def log_title(title: str, title_len: int = TITLE_LEN): ) formatted = f"{SIDE_WHITE_SPACES * ' ' + title + ' ' * SIDE_WHITE_SPACES:=^{final_len}}" - logging.info(formatted) + logging.debug(formatted) @staticmethod def log_to_csv(csv_name: str, field_names: tuple[str], row: dict | None = None): - field_names = list(map(lambda x: x.replace('_', ' '), field_names)) - if row is not None: - row = {k.replace('_', ' '): v for k, v in row.items()} - if isinstance(row, dict): - with open(csv_name, 'a', encoding='utf-8', newline='') as file: - writer = csv.DictWriter(file, fieldnames=field_names) - writer.writerow(row) - elif row is None: - with open(csv_name, 'w', encoding='utf-8', newline='') as file: - writer = csv.writer(file) - writer.writerow(field_names) - else: + if row is not None and not isinstance(row, dict): raise TypeError(f"row has type {type(row)} but must be [ dict | None ]") + field_names = tuple(x.replace('_', ' ') for x in field_names) + cached = logger._csv_files.get(csv_name) + if row is None or cached is None or cached[2] != field_names: + if cached is not None: + cached[0].close() + del logger._csv_files[csv_name] + file = open(csv_name, 'w' if row is None else 'a', encoding='utf-8', newline='') + writer = csv.DictWriter(file, fieldnames=field_names) + logger._csv_files[csv_name] = file, writer, field_names + else: + file, writer, _ = cached + if row is None: + writer.writeheader() + logger._csv_progress[csv_name] = (0, monotonic()) + else: + writer.writerow({k.replace('_', ' '): v for k, v in row.items()}) + file.flush() + if row is not None: + now = monotonic() + count, last_progress = logger._csv_progress.get(csv_name, (0, now)) + count += 1 + if now - last_progress >= 5: + logger._progress_log.info("CSV %s: %d rows written", csv_name, count) + last_progress = now + logger._csv_progress[csv_name] = (count, last_progress) @staticmethod def log_to_stdout(info: dict): - logging.info(f"{info}") + logging.debug("%s", info) @staticmethod def log_sep(): - logging.info("-" * TITLE_LEN) + logging.debug("-" * TITLE_LEN) @staticmethod def log_error(error: str): @@ -60,6 +91,9 @@ def log_warning(warning: str): logging.warning(warning) +atexit.register(logger.close_csv_files) + + def parse_time(datetime_str) -> datetime: start = ( datetime_str[0].split('/') + datetime_str[1].split(':') diff --git a/src/wikipars.py b/src/wikipars.py index ebf21395..e84cecae 100644 --- a/src/wikipars.py +++ b/src/wikipars.py @@ -81,8 +81,7 @@ def wikiparser(repositories: list[str], path_drepo: str, csv_name: str): data_commit["added lines"] = commit.stats.total["insertions"] data_commit["deleted lines"] = commit.stats.total["deletions"] - for fieldname in data_commit: - print(fieldname, data_commit[fieldname], sep=': ') + logger.log_to_stdout(data_commit) logger.log_sep() logger.log_to_csv(csv_name, data_commit)