From a88f0acea63f78b71849731285fc8d38cf535eba Mon Sep 17 00:00:00 2001 From: Junxiang Huang Date: Fri, 29 May 2026 18:55:42 +0800 Subject: [PATCH 1/6] support recording times model usage --- dtable_events/app/config.py | 11 ++++- dtable_events/tasks/ai_stats_worker.py | 63 +++++++++++++++++++------- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/dtable_events/app/config.py b/dtable_events/app/config.py index 0f36c87a..2c8611ac 100644 --- a/dtable_events/app/config.py +++ b/dtable_events/app/config.py @@ -100,7 +100,16 @@ def get_llm_prices(models): # AI models and prices LLM_MODELS = configs.get('LLM_MODELS', []) AI_PRICES = get_llm_prices(LLM_MODELS) -BAIDU_OCR_TOKENS = configs.get('BAIDU_OCR_TOKENS', default={}) + +## OCR +OCR_SERVICE_CONFIG = configs.get('OCR', {}) +OCR_SERVICE_MODEL = OCR_SERVICE_CONFIG.get('model', 'baidu-ocr') +OCR_SERVICE_PRICE = OCR_SERVICE_CONFIG.get('price', 0) +AI_PRICES.update({ + OCR_SERVICE_MODEL: { + 'times': OCR_SERVICE_PRICE + } +}) # IO server ARCHIVE_VIEW_EXPORT_ROW_LIMIT = configs.get('ARCHIVE_VIEW_EXPORT_ROW_LIMIT', default=250000) diff --git a/dtable_events/tasks/ai_stats_worker.py b/dtable_events/tasks/ai_stats_worker.py index b6561ba7..e5913d49 100644 --- a/dtable_events/tasks/ai_stats_worker.py +++ b/dtable_events/tasks/ai_stats_worker.py @@ -10,10 +10,10 @@ from dateutil import relativedelta from sqlalchemy import text -from dtable_events.app.config import AI_PRICES, BAIDU_OCR_TOKENS, AI_STATS_ENABLED +from dtable_events.app.config import AI_PRICES, OCR_SERVICE_MODEL_NAME, OCR_SERVICE_PRICE, AI_STATS_ENABLED from dtable_events.app.event_redis import RedisClient, redis_cache from dtable_events.db import init_db_session_class -from dtable_events.utils import get_opt_from_conf_or_env, parse_bool, uuid_str_to_36_chars, uuid_str_to_32_chars +from dtable_events.utils import uuid_str_to_36_chars, uuid_str_to_32_chars logger = logging.getLogger(__name__) @@ -61,9 +61,6 @@ def save_to_memory(self, usage_info, session): if not isinstance(usage.get('output_tokens'), int): usage['output_tokens'] = 0 - if model in BAIDU_OCR_TOKENS: - usage['output_tokens'] = BAIDU_OCR_TOKENS[model] - if usage_info.get('assistant_uuid'): # for assistant call, set stat object to the assistant owner, i.e., table admin, group admin ... @@ -74,23 +71,28 @@ def save_to_memory(self, usage_info, session): logger.warning('assistant %s has no owner', assistant_uuid) return if owner_info['org_id'] != -1: + self.org_stats[owner_info['org_id']][model]['times'] += usage['times'] self.org_stats[owner_info['org_id']][model]['input_tokens'] += usage.get('input_tokens') or 0 self.org_stats[owner_info['org_id']][model]['output_tokens'] += usage.get('output_tokens') or 0 else: + self.owner_stats[owner_info['owner_id']][model]['times'] += usage['times'] self.owner_stats[owner_info['owner_id']][model]['input_tokens'] += usage.get('input_tokens') or 0 self.owner_stats[owner_info['owner_id']][model]['output_tokens'] += usage.get('output_tokens') or 0 else: # for non-assistant call, set stat obj to common user if usage_info['org_id'] != -1: + self.org_stats[usage_info['org_id']][model]['times'] += usage['times'] self.org_stats[usage_info['org_id']][model]['input_tokens'] += usage.get('input_tokens') or 0 self.org_stats[usage_info['org_id']][model]['output_tokens'] += usage.get('output_tokens') or 0 else: + self.owner_stats[usage_info['username']][model]['times'] += usage['times'] self.owner_stats[usage_info['username']][model]['input_tokens'] += usage.get('input_tokens') or 0 self.owner_stats[usage_info['username']][model]['output_tokens'] += usage.get('output_tokens') or 0 dtable_uuid = usage_info.get('dtable_uuid') if dtable_uuid: dtable_uuid = uuid_str_to_32_chars(usage_info['dtable_uuid']) + self.dtable_stats[dtable_uuid][model]['times'] += usage['times'] self.dtable_stats[dtable_uuid][model]['input_tokens'] += usage.get('input_tokens') or 0 self.dtable_stats[dtable_uuid][model]['output_tokens'] += usage.get('output_tokens') or 0 @@ -190,20 +192,28 @@ def stats_worker(self): team_data = [] team_sql = ''' - INSERT INTO `stats_ai_by_team`(`org_id`, `month`, `model`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:org_id, :month, :model, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) - ON DUPLICATE KEY UPDATE `input_tokens`=`input_tokens`+VALUES(`input_tokens`), + INSERT INTO `stats_ai_by_team`(`org_id`, `month`, `model`, `times`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:org_id, :month, :model, :times, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + ON DUPLICATE KEY UPDATE `times`=`times`+VALUES(`times`), + `input_tokens`=`input_tokens`+VALUES(`input_tokens`), `output_tokens`=`output_tokens`+VALUES(`output_tokens`), `cost`=`cost`+VALUES(`cost`), `updated_at`=VALUES(`updated_at`) ''' for org_id, models_dict in org_stats.items(): for model, usage in models_dict.items(): + # get usage metadata + times = usage.get('times') or 0 input_tokens = usage.get('input_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 + # get price + times_price = AI_PRICES[model].get('times') or 0 input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 + + # calculate cost + times_cost = times_price * times input_cost = input_tokens_price * (input_tokens / 1000000) output_cost = output_tokens_price * (output_tokens / 1000000) logger.info('org %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', org_id, model, input_tokens, input_cost, output_tokens, output_cost) @@ -212,9 +222,10 @@ def stats_worker(self): 'org_id': org_id, 'month': month, 'model': model, + 'times': times, 'input_tokens': input_tokens, 'output_tokens': output_tokens, - 'cost': input_cost + output_cost, + 'cost': times_cost + input_cost + output_cost, 'created_at': datetime.now(), 'updated_at': datetime.now() } @@ -222,20 +233,28 @@ def stats_worker(self): owner_data = [] owner_sql = ''' - INSERT INTO `stats_ai_by_owner`(`owner_id`, `month`, `model`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:owner_id, :month, :model, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) - ON DUPLICATE KEY UPDATE `input_tokens`=`input_tokens`+VALUES(`input_tokens`), + INSERT INTO `stats_ai_by_owner`(`owner_id`, `month`, `model`, `times`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:owner_id, :month, :model, :times, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + ON DUPLICATE KEY UPDATE `times`=`times`+VALUES(`times`), + `input_tokens`=`input_tokens`+VALUES(`input_tokens`), `output_tokens`=`output_tokens`+VALUES(`output_tokens`), `cost`=`cost`+VALUES(`cost`), `updated_at`=VALUES(`updated_at`) ''' for owner_id, models_dict in owner_stats.items(): for model, usage in models_dict.items(): + # get usage metadata + times = usage.get('times') or 0 input_tokens = usage.get('input_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 + # get price + times_price = AI_PRICES[model].get('times') or 0 input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 + + # calculate cost + times_cost = times * times_price input_cost = input_tokens_price * (input_tokens / 1000000) output_cost = output_tokens_price * (output_tokens / 1000000) logger.info('owner %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', owner_id, model, input_tokens, input_cost, output_tokens, output_cost) @@ -244,9 +263,10 @@ def stats_worker(self): 'owner_id': owner_id, 'month': month, 'model': model, + 'times': times, 'input_tokens': input_tokens, 'output_tokens': output_tokens, - 'cost': input_cost + output_cost, + 'cost': times_cost + input_cost + output_cost, 'created_at': datetime.now(), 'updated_at': datetime.now() } @@ -254,9 +274,10 @@ def stats_worker(self): dtable_data = [] dtable_sql = ''' - INSERT INTO `stats_ai_by_dtable`(`dtable_uuid`, `date`, `model`, `owner`, `org_id`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:dtable_uuid, :date, :model, :owner, :org_id, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) - ON DUPLICATE KEY UPDATE `input_tokens`=`input_tokens`+VALUES(`input_tokens`), + INSERT INTO `stats_ai_by_dtable`(`dtable_uuid`, `date`, `model`, `owner`, `org_id`, `times`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:dtable_uuid, :date, :model, :owner, :org_id, :times, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + ON DUPLICATE KEY UPDATE `times`=`times`+VALUES(`times`), + `input_tokens`=`input_tokens`+VALUES(`input_tokens`), `output_tokens`=`output_tokens`+VALUES(`output_tokens`), `cost`=`cost`+VALUES(`cost`), `updated_at`=VALUES(`updated_at`) @@ -264,6 +285,8 @@ def stats_worker(self): dtable_owners_dict = self.query_dtable_owners(list(dtable_stats.keys())) for dtable_uuid, models_dict in dtable_stats.items(): for model, usage in models_dict.items(): + # get usage metadata + times = usage.get('times') or 0 input_tokens = usage.get('input_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 owner_info = dtable_owners_dict.get(dtable_uuid) @@ -274,8 +297,13 @@ def stats_worker(self): owner = None org_id = None + # get price + times_price = AI_PRICES[model].get('times') or 0 input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 + + # calculate cost + times_cost = times * times_price input_cost = input_tokens_price * (input_tokens / 1000000) output_cost = output_tokens_price * (output_tokens / 1000000) logger.info('dtable %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', dtable_uuid, model, input_tokens, input_cost, output_tokens, output_cost) @@ -286,9 +314,10 @@ def stats_worker(self): 'model': model, 'owner': owner, 'org_id': org_id, + 'times': times, 'input_tokens': input_tokens, 'output_tokens': output_tokens, - 'cost': input_cost + output_cost, + 'cost': times_cost + input_cost + output_cost, 'created_at': datetime.now(), 'updated_at': datetime.now() } From 14649d16350390b9e0387d8d768a0a8187a37af8 Mon Sep 17 00:00:00 2001 From: Junxiang Huang Date: Mon, 1 Jun 2026 10:26:11 +0800 Subject: [PATCH 2/6] fix(ai stats): wrong import --- dtable_events/tasks/ai_stats_worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dtable_events/tasks/ai_stats_worker.py b/dtable_events/tasks/ai_stats_worker.py index e5913d49..e251f28c 100644 --- a/dtable_events/tasks/ai_stats_worker.py +++ b/dtable_events/tasks/ai_stats_worker.py @@ -10,7 +10,7 @@ from dateutil import relativedelta from sqlalchemy import text -from dtable_events.app.config import AI_PRICES, OCR_SERVICE_MODEL_NAME, OCR_SERVICE_PRICE, AI_STATS_ENABLED +from dtable_events.app.config import AI_PRICES, AI_STATS_ENABLED from dtable_events.app.event_redis import RedisClient, redis_cache from dtable_events.db import init_db_session_class from dtable_events.utils import uuid_str_to_36_chars, uuid_str_to_32_chars From ffcb92337c9d37e9356fd2acd1c5bc166ddfdbc3 Mon Sep 17 00:00:00 2001 From: Junxiang Huang Date: Mon, 1 Jun 2026 10:32:17 +0800 Subject: [PATCH 3/6] opt: ocr service price config compatible --- dtable_events/app/config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dtable_events/app/config.py b/dtable_events/app/config.py index 2c8611ac..276ccdd1 100644 --- a/dtable_events/app/config.py +++ b/dtable_events/app/config.py @@ -104,10 +104,16 @@ def get_llm_prices(models): ## OCR OCR_SERVICE_CONFIG = configs.get('OCR', {}) OCR_SERVICE_MODEL = OCR_SERVICE_CONFIG.get('model', 'baidu-ocr') -OCR_SERVICE_PRICE = OCR_SERVICE_CONFIG.get('price', 0) +_ocr_service_price = OCR_SERVICE_CONFIG.get('price') +if isinstance(_ocr_service_price, dict): + _ocr_service_price = _ocr_service_price.get('times', 0) +elif _ocr_service_price is None: + _ocr_service_price = 0 +elif not isinstance(_ocr_service_price, (int, float)): + raise ValueError('Invalid ocr service price') AI_PRICES.update({ OCR_SERVICE_MODEL: { - 'times': OCR_SERVICE_PRICE + 'times': _ocr_service_price } }) From 707ed7ba64f4f2d96716e43b191b8805de27712d Mon Sep 17 00:00:00 2001 From: Junxiang Huang Date: Mon, 1 Jun 2026 11:32:18 +0800 Subject: [PATCH 4/6] fix: missing times key --- dtable_events/tasks/ai_stats_worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dtable_events/tasks/ai_stats_worker.py b/dtable_events/tasks/ai_stats_worker.py index e251f28c..939e3673 100644 --- a/dtable_events/tasks/ai_stats_worker.py +++ b/dtable_events/tasks/ai_stats_worker.py @@ -36,9 +36,9 @@ def _parse_config(self): self._enabled = AI_STATS_ENABLED def reset_stats(self): - self.org_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) - self.owner_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) - self.dtable_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) + self.org_stats = defaultdict(lambda: defaultdict(lambda: {'times': 0, 'input_tokens': 0, 'output_tokens': 0})) + self.owner_stats = defaultdict(lambda: defaultdict(lambda: {'times': 0, 'input_tokens': 0, 'output_tokens': 0})) + self.dtable_stats = defaultdict(lambda: defaultdict(lambda: {'times': 0, 'input_tokens': 0, 'output_tokens': 0})) def save_to_memory(self, usage_info, session): if not usage_info.get('model'): From c3d2c60161c86171e0d596581d6c94e4d66891bd Mon Sep 17 00:00:00 2001 From: Junxiang Huang Date: Mon, 1 Jun 2026 11:33:31 +0800 Subject: [PATCH 5/6] fix: missing times key --- dtable_events/tasks/ai_stats_worker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dtable_events/tasks/ai_stats_worker.py b/dtable_events/tasks/ai_stats_worker.py index 939e3673..567de1ea 100644 --- a/dtable_events/tasks/ai_stats_worker.py +++ b/dtable_events/tasks/ai_stats_worker.py @@ -56,6 +56,8 @@ def save_to_memory(self, usage_info, session): if 'completion_tokens' in usage: usage['output_tokens'] = usage['completion_tokens'] + if not isinstance(usage.get('times'), int): + usage['times'] = 0 if not isinstance(usage.get('input_tokens'), int): usage['input_tokens'] = 0 if not isinstance(usage.get('output_tokens'), int): From ae89c3e4f6e4a6d0d9152d1014daa7fb53f557d0 Mon Sep 17 00:00:00 2001 From: Junxiang Huang Date: Mon, 1 Jun 2026 14:00:57 +0800 Subject: [PATCH 6/6] revert: use input_tokens replace times usage --- dtable_events/app/config.py | 17 +----- dtable_events/tasks/ai_stats_worker.py | 77 +++++++++----------------- 2 files changed, 28 insertions(+), 66 deletions(-) diff --git a/dtable_events/app/config.py b/dtable_events/app/config.py index 276ccdd1..d0344b16 100644 --- a/dtable_events/app/config.py +++ b/dtable_events/app/config.py @@ -99,23 +99,12 @@ def get_llm_prices(models): # AI models and prices LLM_MODELS = configs.get('LLM_MODELS', []) -AI_PRICES = get_llm_prices(LLM_MODELS) ## OCR OCR_SERVICE_CONFIG = configs.get('OCR', {}) -OCR_SERVICE_MODEL = OCR_SERVICE_CONFIG.get('model', 'baidu-ocr') -_ocr_service_price = OCR_SERVICE_CONFIG.get('price') -if isinstance(_ocr_service_price, dict): - _ocr_service_price = _ocr_service_price.get('times', 0) -elif _ocr_service_price is None: - _ocr_service_price = 0 -elif not isinstance(_ocr_service_price, (int, float)): - raise ValueError('Invalid ocr service price') -AI_PRICES.update({ - OCR_SERVICE_MODEL: { - 'times': _ocr_service_price - } -}) + +## AI price +AI_PRICES = get_llm_prices(LLM_MODELS + [OCR_SERVICE_CONFIG]) # IO server ARCHIVE_VIEW_EXPORT_ROW_LIMIT = configs.get('ARCHIVE_VIEW_EXPORT_ROW_LIMIT', default=250000) diff --git a/dtable_events/tasks/ai_stats_worker.py b/dtable_events/tasks/ai_stats_worker.py index 567de1ea..e77d04e4 100644 --- a/dtable_events/tasks/ai_stats_worker.py +++ b/dtable_events/tasks/ai_stats_worker.py @@ -36,9 +36,9 @@ def _parse_config(self): self._enabled = AI_STATS_ENABLED def reset_stats(self): - self.org_stats = defaultdict(lambda: defaultdict(lambda: {'times': 0, 'input_tokens': 0, 'output_tokens': 0})) - self.owner_stats = defaultdict(lambda: defaultdict(lambda: {'times': 0, 'input_tokens': 0, 'output_tokens': 0})) - self.dtable_stats = defaultdict(lambda: defaultdict(lambda: {'times': 0, 'input_tokens': 0, 'output_tokens': 0})) + self.org_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) + self.owner_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) + self.dtable_stats = defaultdict(lambda: defaultdict(lambda: {'input_tokens': 0, 'output_tokens': 0})) def save_to_memory(self, usage_info, session): if not usage_info.get('model'): @@ -46,18 +46,23 @@ def save_to_memory(self, usage_info, session): model = usage_info['model'] usage = usage_info.get('usage') or {} + org_id = usage_info.get('org_id') if model not in AI_PRICES: logger.warning('model %s price not defined', model) return + + try: + org_id = int(org_id) + except (TypeError, ValueError): + org_id = -1 + if 'prompt_tokens' in usage: usage['input_tokens'] = usage['prompt_tokens'] if 'completion_tokens' in usage: usage['output_tokens'] = usage['completion_tokens'] - if not isinstance(usage.get('times'), int): - usage['times'] = 0 if not isinstance(usage.get('input_tokens'), int): usage['input_tokens'] = 0 if not isinstance(usage.get('output_tokens'), int): @@ -72,29 +77,24 @@ def save_to_memory(self, usage_info, session): if not owner_info: logger.warning('assistant %s has no owner', assistant_uuid) return - if owner_info['org_id'] != -1: - self.org_stats[owner_info['org_id']][model]['times'] += usage['times'] - self.org_stats[owner_info['org_id']][model]['input_tokens'] += usage.get('input_tokens') or 0 - self.org_stats[owner_info['org_id']][model]['output_tokens'] += usage.get('output_tokens') or 0 + if org_id != -1: + self.org_stats[org_id][model]['input_tokens'] += usage.get('input_tokens') or 0 + self.org_stats[org_id][model]['output_tokens'] += usage.get('output_tokens') or 0 else: - self.owner_stats[owner_info['owner_id']][model]['times'] += usage['times'] self.owner_stats[owner_info['owner_id']][model]['input_tokens'] += usage.get('input_tokens') or 0 self.owner_stats[owner_info['owner_id']][model]['output_tokens'] += usage.get('output_tokens') or 0 else: # for non-assistant call, set stat obj to common user if usage_info['org_id'] != -1: - self.org_stats[usage_info['org_id']][model]['times'] += usage['times'] self.org_stats[usage_info['org_id']][model]['input_tokens'] += usage.get('input_tokens') or 0 self.org_stats[usage_info['org_id']][model]['output_tokens'] += usage.get('output_tokens') or 0 else: - self.owner_stats[usage_info['username']][model]['times'] += usage['times'] self.owner_stats[usage_info['username']][model]['input_tokens'] += usage.get('input_tokens') or 0 self.owner_stats[usage_info['username']][model]['output_tokens'] += usage.get('output_tokens') or 0 dtable_uuid = usage_info.get('dtable_uuid') if dtable_uuid: dtable_uuid = uuid_str_to_32_chars(usage_info['dtable_uuid']) - self.dtable_stats[dtable_uuid][model]['times'] += usage['times'] self.dtable_stats[dtable_uuid][model]['input_tokens'] += usage.get('input_tokens') or 0 self.dtable_stats[dtable_uuid][model]['output_tokens'] += usage.get('output_tokens') or 0 @@ -194,28 +194,20 @@ def stats_worker(self): team_data = [] team_sql = ''' - INSERT INTO `stats_ai_by_team`(`org_id`, `month`, `model`, `times`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:org_id, :month, :model, :times, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) - ON DUPLICATE KEY UPDATE `times`=`times`+VALUES(`times`), - `input_tokens`=`input_tokens`+VALUES(`input_tokens`), + INSERT INTO `stats_ai_by_team`(`org_id`, `month`, `model`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:org_id, :month, :model, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + ON DUPLICATE KEY UPDATE `input_tokens`=`input_tokens`+VALUES(`input_tokens`), `output_tokens`=`output_tokens`+VALUES(`output_tokens`), `cost`=`cost`+VALUES(`cost`), `updated_at`=VALUES(`updated_at`) ''' for org_id, models_dict in org_stats.items(): for model, usage in models_dict.items(): - # get usage metadata - times = usage.get('times') or 0 input_tokens = usage.get('input_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 - # get price - times_price = AI_PRICES[model].get('times') or 0 input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 - - # calculate cost - times_cost = times_price * times input_cost = input_tokens_price * (input_tokens / 1000000) output_cost = output_tokens_price * (output_tokens / 1000000) logger.info('org %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', org_id, model, input_tokens, input_cost, output_tokens, output_cost) @@ -224,10 +216,9 @@ def stats_worker(self): 'org_id': org_id, 'month': month, 'model': model, - 'times': times, 'input_tokens': input_tokens, 'output_tokens': output_tokens, - 'cost': times_cost + input_cost + output_cost, + 'cost': input_cost + output_cost, 'created_at': datetime.now(), 'updated_at': datetime.now() } @@ -235,28 +226,20 @@ def stats_worker(self): owner_data = [] owner_sql = ''' - INSERT INTO `stats_ai_by_owner`(`owner_id`, `month`, `model`, `times`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:owner_id, :month, :model, :times, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) - ON DUPLICATE KEY UPDATE `times`=`times`+VALUES(`times`), - `input_tokens`=`input_tokens`+VALUES(`input_tokens`), + INSERT INTO `stats_ai_by_owner`(`owner_id`, `month`, `model`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:owner_id, :month, :model, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + ON DUPLICATE KEY UPDATE `input_tokens`=`input_tokens`+VALUES(`input_tokens`), `output_tokens`=`output_tokens`+VALUES(`output_tokens`), `cost`=`cost`+VALUES(`cost`), `updated_at`=VALUES(`updated_at`) ''' for owner_id, models_dict in owner_stats.items(): for model, usage in models_dict.items(): - # get usage metadata - times = usage.get('times') or 0 input_tokens = usage.get('input_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 - # get price - times_price = AI_PRICES[model].get('times') or 0 input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 - - # calculate cost - times_cost = times * times_price input_cost = input_tokens_price * (input_tokens / 1000000) output_cost = output_tokens_price * (output_tokens / 1000000) logger.info('owner %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', owner_id, model, input_tokens, input_cost, output_tokens, output_cost) @@ -265,10 +248,9 @@ def stats_worker(self): 'owner_id': owner_id, 'month': month, 'model': model, - 'times': times, 'input_tokens': input_tokens, 'output_tokens': output_tokens, - 'cost': times_cost + input_cost + output_cost, + 'cost': input_cost + output_cost, 'created_at': datetime.now(), 'updated_at': datetime.now() } @@ -276,10 +258,9 @@ def stats_worker(self): dtable_data = [] dtable_sql = ''' - INSERT INTO `stats_ai_by_dtable`(`dtable_uuid`, `date`, `model`, `owner`, `org_id`, `times`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) - VALUES (:dtable_uuid, :date, :model, :owner, :org_id, :times, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) - ON DUPLICATE KEY UPDATE `times`=`times`+VALUES(`times`), - `input_tokens`=`input_tokens`+VALUES(`input_tokens`), + INSERT INTO `stats_ai_by_dtable`(`dtable_uuid`, `date`, `model`, `owner`, `org_id`, `input_tokens`, `output_tokens`, `cost`, `created_at`, `updated_at`) + VALUES (:dtable_uuid, :date, :model, :owner, :org_id, :input_tokens, :output_tokens, :cost, :created_at, :updated_at) + ON DUPLICATE KEY UPDATE `input_tokens`=`input_tokens`+VALUES(`input_tokens`), `output_tokens`=`output_tokens`+VALUES(`output_tokens`), `cost`=`cost`+VALUES(`cost`), `updated_at`=VALUES(`updated_at`) @@ -287,8 +268,6 @@ def stats_worker(self): dtable_owners_dict = self.query_dtable_owners(list(dtable_stats.keys())) for dtable_uuid, models_dict in dtable_stats.items(): for model, usage in models_dict.items(): - # get usage metadata - times = usage.get('times') or 0 input_tokens = usage.get('input_tokens') or 0 output_tokens = usage.get('output_tokens') or 0 owner_info = dtable_owners_dict.get(dtable_uuid) @@ -299,13 +278,8 @@ def stats_worker(self): owner = None org_id = None - # get price - times_price = AI_PRICES[model].get('times') or 0 input_tokens_price = AI_PRICES[model].get('input_tokens') or 0 output_tokens_price = AI_PRICES[model].get('output_tokens') or 0 - - # calculate cost - times_cost = times * times_price input_cost = input_tokens_price * (input_tokens / 1000000) output_cost = output_tokens_price * (output_tokens / 1000000) logger.info('dtable %s model %s, input_tokens %s cost %s, output_tokens %s cost %s', dtable_uuid, model, input_tokens, input_cost, output_tokens, output_cost) @@ -316,10 +290,9 @@ def stats_worker(self): 'model': model, 'owner': owner, 'org_id': org_id, - 'times': times, 'input_tokens': input_tokens, 'output_tokens': output_tokens, - 'cost': times_cost + input_cost + output_cost, + 'cost': input_cost + output_cost, 'created_at': datetime.now(), 'updated_at': datetime.now() }