From 1df408b7c303d951acfcac7fd81de3d6e8fdfd4f Mon Sep 17 00:00:00 2001 From: Christopher Date: Sat, 31 Jan 2026 09:50:10 -0500 Subject: [PATCH 1/4] Fixed score and api call failing with retries --- fetch_bitcoin_data.py | 138 ++++++++++++++++++++++------- metrics/puell_multiple.py | 15 ++-- metrics/two_year_moving_average.py | 13 +-- utils.py | 54 ++++++++++- 4 files changed, 168 insertions(+), 52 deletions(-) diff --git a/fetch_bitcoin_data.py b/fetch_bitcoin_data.py index c226f88..55b83b7 100644 --- a/fetch_bitcoin_data.py +++ b/fetch_bitcoin_data.py @@ -2,7 +2,102 @@ import pandas as pd from filecache import filecache -from utils import HTTP, mark_days_since, mark_highs_lows +from utils import HTTP, http_get_with_retry, mark_days_since, mark_highs_lows + +# Known Bitcoin halving dates and block heights for accurate calculations +HALVING_DATA = [ + # (date, block_height, block_reward) + ('2009-01-03', 0, 50.0), # Genesis block + ('2012-11-28', 210000, 25.0), # 1st halving + ('2016-07-09', 420000, 12.5), # 2nd halving + ('2020-05-11', 630000, 6.25), # 3rd halving + ('2024-04-20', 840000, 3.125), # 4th halving +] + + +def fetch_blockchain_data() -> pd.DataFrame: + """ + Fetches historical Bitcoin blockchain data from Blockchain.com API. + Uses miners-revenue chart for USD mining revenue. + Block heights and BTC generation are calculated from halving schedule. + + Returns: + DataFrame with Date, TotalBlocks, MinBlockID, MaxBlockID, + TotalGeneration, TotalGenerationUSD columns. + """ + # Fetch mining revenue from Blockchain.com (free, reliable) + response = http_get_with_retry( + 'https://api.blockchain.info/charts/miners-revenue', + params={ + 'timespan': 'all', + 'format': 'json', + 'sampled': 'false', + }, + ) + revenue_data = response.json() + + # Create DataFrame from mining revenue data + df = pd.DataFrame(revenue_data['values']) + df.columns = ['DateTimestamp', 'TotalGenerationUSD'] + df['Date'] = pd.to_datetime(df['DateTimestamp'], unit='s').dt.floor('d') + + # Calculate block heights based on known halving dates + # Average ~144 blocks per day (one block every 10 minutes) + genesis_date = pd.Timestamp('2009-01-03') + + # Create halving schedule DataFrame for interpolation + halving_df = pd.DataFrame(HALVING_DATA, columns=['Date', 'BlockHeight', 'BlockReward']) + halving_df['Date'] = pd.to_datetime(halving_df['Date']) + + # Calculate approximate block height for each day using linear interpolation + # between known halving points + def estimate_block_height(date): + date = pd.Timestamp(date) + if date < genesis_date: + return 0 + + # Find the halving period this date falls into + for i in range(len(HALVING_DATA) - 1): + start_date = pd.Timestamp(HALVING_DATA[i][0]) + end_date = pd.Timestamp(HALVING_DATA[i + 1][0]) + start_height = HALVING_DATA[i][1] + end_height = HALVING_DATA[i + 1][1] + + if start_date <= date < end_date: + # Linear interpolation within this halving period + total_days = (end_date - start_date).days + days_elapsed = (date - start_date).days + height = start_height + (end_height - start_height) * days_elapsed / total_days + return int(height) + + # After the last known halving, extrapolate at ~144 blocks/day + last_date = pd.Timestamp(HALVING_DATA[-1][0]) + last_height = HALVING_DATA[-1][1] + days_since = (date - last_date).days + return int(last_height + days_since * 144) + + def get_block_reward(block_height): + """Get block reward for a given block height.""" + halving_interval = 210000 + halvings = block_height // halving_interval + return 50.0 / (2 ** halvings) + + # Calculate block data for each day + df['MaxBlockID'] = df['Date'].apply(estimate_block_height) + df['MinBlockID'] = df['MaxBlockID'].shift(1).fillna(0).astype(int) + df['TotalBlocks'] = df['MaxBlockID'] - df['MinBlockID'] + df['TotalBlocks'] = df['TotalBlocks'].clip(lower=1) # Ensure at least 1 block + + # Calculate BTC generation based on block reward + # Store in satoshis (multiply by 1e8) to match original Blockchair format + df['BlockReward'] = df['MaxBlockID'].apply(get_block_reward) + df['TotalGeneration'] = df['TotalBlocks'] * df['BlockReward'] * 1e8 # Convert to satoshis + + # Select and order columns to match original format + df = df[['Date', 'TotalBlocks', 'MinBlockID', 'MaxBlockID', 'TotalGeneration', 'TotalGenerationUSD']] + df = df.sort_values('Date').reset_index(drop=True) + + return df @filecache(7200) # 2 hours @@ -16,28 +111,8 @@ def fetch_bitcoin_data() -> pd.DataFrame: """ print('📈 Requesting historical Bitcoin data…') - response = HTTP.get( - 'https://api.blockchair.com/bitcoin/blocks', - params={ - 'a': 'date,count(),min(id),max(id),sum(generation),sum(generation_usd)', - 's': 'date(desc)', - }, - ) - response.raise_for_status() - response_json = response.json() - - df = pd.DataFrame(response_json['data'][::-1]) - df.rename( - columns={ - 'date': 'Date', - 'count()': 'TotalBlocks', - 'min(id)': 'MinBlockID', - 'max(id)': 'MaxBlockID', - 'sum(generation)': 'TotalGeneration', - 'sum(generation_usd)': 'TotalGenerationUSD', - }, - inplace=True, - ) + # Use Blockchain.com API instead of Blockchair (which is blocked) + df = fetch_blockchain_data() df['Date'] = pd.to_datetime(df['Date']) df['TotalGeneration'] /= 1e8 @@ -69,7 +144,7 @@ def fetch_bitcoin_data() -> pd.DataFrame: def fetch_price_data() -> pd.DataFrame: - response = HTTP.get( + response = http_get_with_retry( 'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/detail/chart', params={ 'id': 1, @@ -77,7 +152,6 @@ def fetch_price_data() -> pd.DataFrame: }, ) - response.raise_for_status() response_json = response.json() response_x = [float(k) for k in response_json['data']['points']] response_y = [value['v'][0] for value in response_json['data']['points'].values()] @@ -121,17 +195,21 @@ def add_block_halving_data(df: pd.DataFrame) -> pd.DataFrame: 'BlockGeneration', ] = current_block_production - block_halving_row = df[ + block_halving_rows = df[ (df['MinBlockID'] <= current_block_halving_id) & (df['MaxBlockID'] >= current_block_halving_id) - ].squeeze() + ] - if block_halving_row.shape[0] == 0: + if len(block_halving_rows) == 0: break + # Take the first matching row if multiple match + block_halving_row = block_halving_rows.iloc[0] + row_index = block_halving_rows.index[0] + current_block_halving_id += reward_halving_every current_block_production /= 2 - df.loc[block_halving_row.name, 'Halving'] = 1 - df.loc[df.index > block_halving_row.name, 'NextHalvingBlock'] = current_block_halving_id + df.loc[row_index, 'Halving'] = 1 + df.loc[df.index > row_index, 'NextHalvingBlock'] = current_block_halving_id df['DaysToHalving'] = pd.to_timedelta((df['NextHalvingBlock'] - df['MaxBlockID']) / (24 * 6), unit='D') df['NextHalvingDate'] = df['Date'] + df['DaysToHalving'] diff --git a/metrics/puell_multiple.py b/metrics/puell_multiple.py index 50d31b9..1d8fa37 100644 --- a/metrics/puell_multiple.py +++ b/metrics/puell_multiple.py @@ -4,7 +4,6 @@ from matplotlib.axes import Axes from sklearn.linear_model import LinearRegression -from api.coinsoto_api import cs_fetch from metrics.base_metric import BaseMetric from utils import add_common_markers @@ -19,15 +18,11 @@ def description(self) -> str: return 'Puell Multiple' def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: - df = df.merge( - cs_fetch( - path='getPuellMultiple', - data_selector='puellMultiplList', - col_name='Puell', - ), - on='Date', - how='left', - ) + # Calculate Puell Multiple locally from mining revenue data + # Puell = daily_mining_revenue / 365-day_MA_of_mining_revenue + # TotalGenerationUSD contains daily mining revenue in USD from Blockchain.com + df['MiningRevenue365MA'] = df['TotalGenerationUSD'].rolling(window=365, min_periods=1).mean() + df['Puell'] = df['TotalGenerationUSD'] / df['MiningRevenue365MA'] df['Puell'] = df['Puell'].ffill() df['PuellLog'] = np.log(df['Puell']) diff --git a/metrics/two_year_moving_average.py b/metrics/two_year_moving_average.py index eb002fd..ec225f4 100644 --- a/metrics/two_year_moving_average.py +++ b/metrics/two_year_moving_average.py @@ -4,7 +4,6 @@ from matplotlib.axes import Axes from sklearn.linear_model import LinearRegression -from api.coinsoto_api import cs_fetch from metrics.base_metric import BaseMetric from utils import add_common_markers @@ -19,15 +18,9 @@ def description(self) -> str: return '2 Year Moving Average' def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: - df = df.merge( - cs_fetch( - path='getBtcMultiplier', - data_selector='mA730List', - col_name='2YMA', - ), - on='Date', - how='left', - ) + # Calculate 2-year (730-day) moving average locally from price data + # No external API needed - we have the price data already + df['2YMA'] = df['Price'].rolling(window=730, min_periods=1).mean() df['2YMA'] = df['2YMA'].ffill() df['2YMALog'] = np.log(df['2YMA']) df['2YMALogDiff'] = df['PriceLog'] - df['2YMALog'] diff --git a/utils.py b/utils.py index fce7efa..1be3e9e 100644 --- a/utils.py +++ b/utils.py @@ -1,23 +1,73 @@ import os +import time import traceback from datetime import datetime from math import ceil +from functools import wraps import numpy as np import pandas as pd import seaborn as sns import telegram -from httpx import Client +from httpx import Client, HTTPStatusError from matplotlib.axes import Axes from sty import bg +# HTTP client with browser-like headers to reduce blocking HTTP = Client( - headers={'User-Agent': 'Mozilla/5.0 (Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0'}, + headers={ + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'application/json, text/plain, */*', + 'Accept-Language': 'en-US,en;q=0.9', + }, timeout=30, follow_redirects=True, ) +def http_get_with_retry(url: str, params: dict = None, max_retries: int = 3, delay: float = 1.0): + """ + Make an HTTP GET request with retry logic and exponential backoff. + + Args: + url: The URL to fetch + params: Optional query parameters + max_retries: Maximum number of retry attempts + delay: Initial delay between retries (doubles each attempt) + + Returns: + The HTTP response object + + Raises: + HTTPStatusError: If all retries fail + """ + last_error = None + + for attempt in range(max_retries): + try: + response = HTTP.get(url, params=params) + response.raise_for_status() + return response + except HTTPStatusError as e: + last_error = e + # Don't retry on client errors (4xx) except rate limiting (429) + if 400 <= e.response.status_code < 500 and e.response.status_code != 429: + raise + # Wait before retry with exponential backoff + if attempt < max_retries - 1: + wait_time = delay * (2 ** attempt) + time.sleep(wait_time) + except Exception as e: + last_error = e + if attempt < max_retries - 1: + wait_time = delay * (2 ** attempt) + time.sleep(wait_time) + + if last_error: + raise last_error + raise Exception(f"Failed to fetch {url} after {max_retries} attempts") + + def mark_highs_lows( df: pd.DataFrame, col: str, From 27274420b4c856f58d6f675a513f5bf02dff7775 Mon Sep 17 00:00:00 2001 From: Christopher Date: Sat, 31 Jan 2026 09:53:17 -0500 Subject: [PATCH 2/4] remove excessing logging --- api/cbbiinfo_api.py | 3 ++- metrics/base_metric.py | 3 +-- metrics/rhodl_ratio.py | 4 +--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/api/cbbiinfo_api.py b/api/cbbiinfo_api.py index 2ba9590..36745f2 100644 --- a/api/cbbiinfo_api.py +++ b/api/cbbiinfo_api.py @@ -15,6 +15,7 @@ def cbbi_fetch(key: str) -> pd.DataFrame: 'Value', ], ) - df['Date'] = pd.to_datetime(df['Date'], unit='s').dt.tz_localize(None) + # Convert string timestamps to int before datetime conversion to avoid FutureWarning + df['Date'] = pd.to_datetime(df['Date'].astype(int), unit='s').dt.tz_localize(None) return df diff --git a/metrics/base_metric.py b/metrics/base_metric.py index 71866da..ae2db2e 100644 --- a/metrics/base_metric.py +++ b/metrics/base_metric.py @@ -1,4 +1,3 @@ -import traceback from abc import ABC, abstractmethod import pandas as pd @@ -34,7 +33,7 @@ async def calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: try: return self._calculate(df, ax) except Exception as ex: - traceback.print_exc() + # Silently fall back - no traceback printed to keep logs clean await send_error_notification(ex) print(fg.black + bg.yellow + f' Requesting fallback values for {self.name} (from CBBI.info) ' + rs.all) diff --git a/metrics/rhodl_ratio.py b/metrics/rhodl_ratio.py index c77bef5..662ca52 100644 --- a/metrics/rhodl_ratio.py +++ b/metrics/rhodl_ratio.py @@ -1,5 +1,3 @@ -import traceback - import numpy as np import pandas as pd import seaborn as sns @@ -30,7 +28,7 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: col_name='RHODL', ) except Exception: - traceback.print_exc() + # Silently try GlassNode fallback - no traceback printed print(fg.black + bg.yellow + f' Requesting fallback values for {self.name} (from GlassNode) ' + rs.all) remote_df = gn_fetch(url_selector='rhodl_ratio', col_name='RHODL', a='BTC') From 2484b1208f83136a932e9a3eb13b967a342cd616 Mon Sep 17 00:00:00 2001 From: Kamil Monicz Date: Sun, 1 Feb 2026 15:32:00 +0000 Subject: [PATCH 3/4] Undo unnecessary changes and fix coinank endpoint --- api/cbbiinfo_api.py | 3 +- api/coinsoto_api.py | 12 ++- fetch_bitcoin_data.py | 162 +++++++++++++++----------------------- metrics/base_metric.py | 3 +- metrics/pi_cycle.py | 4 + metrics/puell_multiple.py | 3 +- metrics/rhodl_ratio.py | 4 +- utils.py | 56 +------------ 8 files changed, 83 insertions(+), 164 deletions(-) diff --git a/api/cbbiinfo_api.py b/api/cbbiinfo_api.py index 36745f2..2ba9590 100644 --- a/api/cbbiinfo_api.py +++ b/api/cbbiinfo_api.py @@ -15,7 +15,6 @@ def cbbi_fetch(key: str) -> pd.DataFrame: 'Value', ], ) - # Convert string timestamps to int before datetime conversion to avoid FutureWarning - df['Date'] = pd.to_datetime(df['Date'].astype(int), unit='s').dt.tz_localize(None) + df['Date'] = pd.to_datetime(df['Date'], unit='s').dt.tz_localize(None) return df diff --git a/api/coinsoto_api.py b/api/coinsoto_api.py index ed2c461..360d274 100644 --- a/api/coinsoto_api.py +++ b/api/coinsoto_api.py @@ -4,7 +4,7 @@ def cs_fetch(path: str, data_selector: str, col_name: str) -> pd.DataFrame: - response = HTTP.get(f'https://coinank.com/indicatorapi/{path}') + response = HTTP.get(f'https://api.coinank.com/indicatorapi/{path}') response.raise_for_status() data = response.json()['data'] @@ -15,12 +15,10 @@ def cs_fetch(path: str, data_selector: str, col_name: str) -> pd.DataFrame: data_y = data[data_selector] assert len(data_x) == len(data_y), f'{len(data_x)=} != {len(data_y)=}' - df = pd.DataFrame( - { - 'Date': data_x[: len(data_y)], - col_name: data_y, - } - ) + df = pd.DataFrame({ + 'Date': data_x[: len(data_y)], + col_name: data_y, + }) df['Date'] = pd.to_datetime(df['Date'], unit='ms').dt.tz_localize(None) diff --git a/fetch_bitcoin_data.py b/fetch_bitcoin_data.py index 55b83b7..24316da 100644 --- a/fetch_bitcoin_data.py +++ b/fetch_bitcoin_data.py @@ -1,18 +1,36 @@ +from itertools import count, pairwise + import numpy as np import pandas as pd from filecache import filecache -from utils import HTTP, http_get_with_retry, mark_days_since, mark_highs_lows +from utils import HTTP, mark_days_since, mark_highs_lows + +HALVING_INTERVAL = 210_000 +GENESIS_BLOCK_REWARD = 50.0 +BLOCKS_PER_DAY = 144 + + +def fetch_block_halving(): + """ + Fetch Bitcoin halving data by probing raw blocks at halving heights. + """ + halving_data: list[tuple[pd.Timestamp, int, float]] = [] + + for halving_index in count(): + block_height = halving_index * HALVING_INTERVAL + block_reward = GENESIS_BLOCK_REWARD / (2**halving_index) + + response = HTTP.get(f'https://blockchain.info/rawblock/{block_height}') + if response.status_code == 404: + break -# Known Bitcoin halving dates and block heights for accurate calculations -HALVING_DATA = [ - # (date, block_height, block_reward) - ('2009-01-03', 0, 50.0), # Genesis block - ('2012-11-28', 210000, 25.0), # 1st halving - ('2016-07-09', 420000, 12.5), # 2nd halving - ('2020-05-11', 630000, 6.25), # 3rd halving - ('2024-04-20', 840000, 3.125), # 4th halving -] + response.raise_for_status() + block_time = response.json()['time'] + block_date = pd.to_datetime(block_time, unit='s').tz_localize(None).floor('d') + halving_data.append((block_date, block_height, block_reward)) + + return halving_data def fetch_blockchain_data() -> pd.DataFrame: @@ -20,13 +38,15 @@ def fetch_blockchain_data() -> pd.DataFrame: Fetches historical Bitcoin blockchain data from Blockchain.com API. Uses miners-revenue chart for USD mining revenue. Block heights and BTC generation are calculated from halving schedule. - + Returns: - DataFrame with Date, TotalBlocks, MinBlockID, MaxBlockID, + DataFrame with Date, TotalBlocks, MinBlockID, MaxBlockID, TotalGeneration, TotalGenerationUSD columns. """ - # Fetch mining revenue from Blockchain.com (free, reliable) - response = http_get_with_retry( + halving_data = fetch_block_halving() + + # Fetch mining revenue from Blockchain.com + response = HTTP.get( 'https://api.blockchain.info/charts/miners-revenue', params={ 'timespan': 'all', @@ -34,69 +54,53 @@ def fetch_blockchain_data() -> pd.DataFrame: 'sampled': 'false', }, ) - revenue_data = response.json() - + # Create DataFrame from mining revenue data - df = pd.DataFrame(revenue_data['values']) + response.raise_for_status() + df = pd.DataFrame(response.json()['values']) df.columns = ['DateTimestamp', 'TotalGenerationUSD'] df['Date'] = pd.to_datetime(df['DateTimestamp'], unit='s').dt.floor('d') - - # Calculate block heights based on known halving dates - # Average ~144 blocks per day (one block every 10 minutes) - genesis_date = pd.Timestamp('2009-01-03') - - # Create halving schedule DataFrame for interpolation - halving_df = pd.DataFrame(HALVING_DATA, columns=['Date', 'BlockHeight', 'BlockReward']) - halving_df['Date'] = pd.to_datetime(halving_df['Date']) - + # Calculate approximate block height for each day using linear interpolation # between known halving points - def estimate_block_height(date): - date = pd.Timestamp(date) - if date < genesis_date: - return 0 - + def estimate_block_height(date: pd.Timestamp): # Find the halving period this date falls into - for i in range(len(HALVING_DATA) - 1): - start_date = pd.Timestamp(HALVING_DATA[i][0]) - end_date = pd.Timestamp(HALVING_DATA[i + 1][0]) - start_height = HALVING_DATA[i][1] - end_height = HALVING_DATA[i + 1][1] - + for (start_date, start_height, _), (end_date, end_height, _) in pairwise(halving_data): if start_date <= date < end_date: # Linear interpolation within this halving period total_days = (end_date - start_date).days days_elapsed = (date - start_date).days height = start_height + (end_height - start_height) * days_elapsed / total_days return int(height) - - # After the last known halving, extrapolate at ~144 blocks/day - last_date = pd.Timestamp(HALVING_DATA[-1][0]) - last_height = HALVING_DATA[-1][1] + + # After the last known halving, extrapolate + (last_date, last_height, _) = halving_data[-1] days_since = (date - last_date).days - return int(last_height + days_since * 144) - + return int(last_height + days_since * BLOCKS_PER_DAY) + def get_block_reward(block_height): """Get block reward for a given block height.""" - halving_interval = 210000 - halvings = block_height // halving_interval - return 50.0 / (2 ** halvings) - + halvings = block_height // HALVING_INTERVAL + return GENESIS_BLOCK_REWARD / (2**halvings) + # Calculate block data for each day df['MaxBlockID'] = df['Date'].apply(estimate_block_height) - df['MinBlockID'] = df['MaxBlockID'].shift(1).fillna(0).astype(int) + df['MinBlockID'] = df['MaxBlockID'].shift(1, fill_value=0) df['TotalBlocks'] = df['MaxBlockID'] - df['MinBlockID'] - df['TotalBlocks'] = df['TotalBlocks'].clip(lower=1) # Ensure at least 1 block - + # Calculate BTC generation based on block reward - # Store in satoshis (multiply by 1e8) to match original Blockchair format df['BlockReward'] = df['MaxBlockID'].apply(get_block_reward) df['TotalGeneration'] = df['TotalBlocks'] * df['BlockReward'] * 1e8 # Convert to satoshis - + # Select and order columns to match original format df = df[['Date', 'TotalBlocks', 'MinBlockID', 'MaxBlockID', 'TotalGeneration', 'TotalGenerationUSD']] df = df.sort_values('Date').reset_index(drop=True) - + + # Add halving markers + df['Halving'] = 0 + for _, block_height, _ in halving_data[1:]: + df.loc[(df['MinBlockID'] < block_height) & (df['MaxBlockID'] >= block_height), 'Halving'] = 1 + return df @@ -111,7 +115,6 @@ def fetch_bitcoin_data() -> pd.DataFrame: """ print('📈 Requesting historical Bitcoin data…') - # Use Blockchain.com API instead of Blockchair (which is blocked) df = fetch_blockchain_data() df['Date'] = pd.to_datetime(df['Date']) @@ -132,7 +135,6 @@ def fetch_bitcoin_data() -> pd.DataFrame: df.reset_index(drop=True, inplace=True) df = fix_current_day_data(df) - df = add_block_halving_data(df) df = mark_highs_lows(df, 'Price', False, round(365 * 2), 180) # move 2021' peak to the first price peak @@ -144,7 +146,7 @@ def fetch_bitcoin_data() -> pd.DataFrame: def fetch_price_data() -> pd.DataFrame: - response = http_get_with_retry( + response = HTTP.get( 'https://api.coinmarketcap.com/data-api/v3/cryptocurrency/detail/chart', params={ 'id': 1, @@ -152,16 +154,15 @@ def fetch_price_data() -> pd.DataFrame: }, ) + response.raise_for_status() response_json = response.json() response_x = [float(k) for k in response_json['data']['points']] response_y = [value['v'][0] for value in response_json['data']['points'].values()] - df = pd.DataFrame( - { - 'Date': response_x, - 'Price': response_y, - } - ) + df = pd.DataFrame({ + 'Date': response_x, + 'Price': response_y, + }) df['Date'] = pd.to_datetime(df['Date'], unit='s').dt.tz_localize(None).dt.floor('d') df.sort_values(by='Date', inplace=True) df.drop_duplicates('Date', keep='last', inplace=True) @@ -172,45 +173,10 @@ def fetch_price_data() -> pd.DataFrame: def fix_current_day_data(df: pd.DataFrame) -> pd.DataFrame: row = df.iloc[-1].copy() - target_total_blocks = 24 * 6 - target_scale = target_total_blocks / row['TotalBlocks'] + target_scale = BLOCKS_PER_DAY / row['TotalBlocks'] for col_name in ['TotalBlocks', 'TotalGeneration', 'TotalGenerationUSD']: row[col_name] *= target_scale df.iloc[-1] = row return df - - -def add_block_halving_data(df: pd.DataFrame) -> pd.DataFrame: - reward_halving_every = 210000 - current_block_halving_id = reward_halving_every - current_block_production = 50 - df['Halving'] = 0 - df['NextHalvingBlock'] = current_block_halving_id - - while True: - df.loc[ - (current_block_halving_id - reward_halving_every) <= df['MaxBlockID'], - 'BlockGeneration', - ] = current_block_production - - block_halving_rows = df[ - (df['MinBlockID'] <= current_block_halving_id) & (df['MaxBlockID'] >= current_block_halving_id) - ] - - if len(block_halving_rows) == 0: - break - - # Take the first matching row if multiple match - block_halving_row = block_halving_rows.iloc[0] - row_index = block_halving_rows.index[0] - - current_block_halving_id += reward_halving_every - current_block_production /= 2 - df.loc[row_index, 'Halving'] = 1 - df.loc[df.index > row_index, 'NextHalvingBlock'] = current_block_halving_id - - df['DaysToHalving'] = pd.to_timedelta((df['NextHalvingBlock'] - df['MaxBlockID']) / (24 * 6), unit='D') - df['NextHalvingDate'] = df['Date'] + df['DaysToHalving'] - return df diff --git a/metrics/base_metric.py b/metrics/base_metric.py index ae2db2e..71866da 100644 --- a/metrics/base_metric.py +++ b/metrics/base_metric.py @@ -1,3 +1,4 @@ +import traceback from abc import ABC, abstractmethod import pandas as pd @@ -33,7 +34,7 @@ async def calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: try: return self._calculate(df, ax) except Exception as ex: - # Silently fall back - no traceback printed to keep logs clean + traceback.print_exc() await send_error_notification(ex) print(fg.black + bg.yellow + f' Requesting fallback values for {self.name} (from CBBI.info) ' + rs.all) diff --git a/metrics/pi_cycle.py b/metrics/pi_cycle.py index 424f451..24a6dfc 100644 --- a/metrics/pi_cycle.py +++ b/metrics/pi_cycle.py @@ -63,4 +63,8 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: sns.lineplot(data=df, x='Date', y='PiCycleIndexNoNa', ax=ax[0]) add_common_markers(df, ax[0]) + sns.lineplot(data=df, x='Date', y='PiCycleDiff', ax=ax[1]) + sns.lineplot(data=df, x='Date', y='PiCycleDiffThreshold', ax=ax[1], linestyle='--') + add_common_markers(df, ax[1], price_line=False) + return df['PiCycleIndex'] diff --git a/metrics/puell_multiple.py b/metrics/puell_multiple.py index 1d8fa37..06aac76 100644 --- a/metrics/puell_multiple.py +++ b/metrics/puell_multiple.py @@ -21,8 +21,7 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: # Calculate Puell Multiple locally from mining revenue data # Puell = daily_mining_revenue / 365-day_MA_of_mining_revenue # TotalGenerationUSD contains daily mining revenue in USD from Blockchain.com - df['MiningRevenue365MA'] = df['TotalGenerationUSD'].rolling(window=365, min_periods=1).mean() - df['Puell'] = df['TotalGenerationUSD'] / df['MiningRevenue365MA'] + df['Puell'] = df['TotalGenerationUSD'] / df['TotalGenerationUSD'].rolling(window=365, min_periods=1).mean() df['Puell'] = df['Puell'].ffill() df['PuellLog'] = np.log(df['Puell']) diff --git a/metrics/rhodl_ratio.py b/metrics/rhodl_ratio.py index 662ca52..c77bef5 100644 --- a/metrics/rhodl_ratio.py +++ b/metrics/rhodl_ratio.py @@ -1,3 +1,5 @@ +import traceback + import numpy as np import pandas as pd import seaborn as sns @@ -28,7 +30,7 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: col_name='RHODL', ) except Exception: - # Silently try GlassNode fallback - no traceback printed + traceback.print_exc() print(fg.black + bg.yellow + f' Requesting fallback values for {self.name} (from GlassNode) ' + rs.all) remote_df = gn_fetch(url_selector='rhodl_ratio', col_name='RHODL', a='BTC') diff --git a/utils.py b/utils.py index 1be3e9e..e19b85d 100644 --- a/utils.py +++ b/utils.py @@ -1,73 +1,23 @@ import os -import time import traceback from datetime import datetime from math import ceil -from functools import wraps import numpy as np import pandas as pd import seaborn as sns import telegram -from httpx import Client, HTTPStatusError +from httpx import Client from matplotlib.axes import Axes from sty import bg -# HTTP client with browser-like headers to reduce blocking HTTP = Client( - headers={ - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Accept': 'application/json, text/plain, */*', - 'Accept-Language': 'en-US,en;q=0.9', - }, + headers={'User-Agent': 'Mozilla/5.0 (Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0'}, timeout=30, follow_redirects=True, ) -def http_get_with_retry(url: str, params: dict = None, max_retries: int = 3, delay: float = 1.0): - """ - Make an HTTP GET request with retry logic and exponential backoff. - - Args: - url: The URL to fetch - params: Optional query parameters - max_retries: Maximum number of retry attempts - delay: Initial delay between retries (doubles each attempt) - - Returns: - The HTTP response object - - Raises: - HTTPStatusError: If all retries fail - """ - last_error = None - - for attempt in range(max_retries): - try: - response = HTTP.get(url, params=params) - response.raise_for_status() - return response - except HTTPStatusError as e: - last_error = e - # Don't retry on client errors (4xx) except rate limiting (429) - if 400 <= e.response.status_code < 500 and e.response.status_code != 429: - raise - # Wait before retry with exponential backoff - if attempt < max_retries - 1: - wait_time = delay * (2 ** attempt) - time.sleep(wait_time) - except Exception as e: - last_error = e - if attempt < max_retries - 1: - wait_time = delay * (2 ** attempt) - time.sleep(wait_time) - - if last_error: - raise last_error - raise Exception(f"Failed to fetch {url} after {max_retries} attempts") - - def mark_highs_lows( df: pd.DataFrame, col: str, @@ -287,4 +237,4 @@ async def send_error_notification(exception: Exception) -> bool: f'
{"".join(traceback.format_exception(exception))}
', parse_mode='HTML', ) - return True + return True \ No newline at end of file From aed29aa7c4c1c1d3f3577ec6e8e34a8b026c34ef Mon Sep 17 00:00:00 2001 From: Emirhan TALA Date: Mon, 2 Feb 2026 11:53:02 +0300 Subject: [PATCH 4/4] fix: Puell Multiple indicator range regression (#22) --- metrics/puell_multiple.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/metrics/puell_multiple.py b/metrics/puell_multiple.py index 06aac76..75cffca 100644 --- a/metrics/puell_multiple.py +++ b/metrics/puell_multiple.py @@ -5,7 +5,7 @@ from sklearn.linear_model import LinearRegression from metrics.base_metric import BaseMetric -from utils import add_common_markers +from utils import add_common_markers, mark_highs_lows class PuellMetric(BaseMetric): @@ -25,7 +25,9 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: df['Puell'] = df['Puell'].ffill() df['PuellLog'] = np.log(df['Puell']) - high_rows = df.loc[df['PriceHigh'] == 1] + df = mark_highs_lows(df, 'PuellLog', True, round(365 * 2), 365) + high_rows = df.loc[(df['PuellLogHigh'] == 1) & (df.index > 365)] + high_x = high_rows.index.values.reshape(-1, 1) high_y = high_rows['PuellLog'].values.reshape(-1, 1) @@ -37,7 +39,9 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: lin_model = LinearRegression() lin_model.fit(high_x, high_y) - df['PuellLogHighModel'] = lin_model.predict(x) + predictions = lin_model.predict(x) + min_peak = high_y.min() + df['PuellLogHighModel'] = np.maximum(predictions, min_peak) # lin_model.fit(low_x, low_y) # df['PuellLogLowModel'] = lin_model.predict(x)