diff --git a/api/coinsoto_api.py b/api/coinsoto_api.py index ed2c461..360d274 100644 --- a/api/coinsoto_api.py +++ b/api/coinsoto_api.py @@ -4,7 +4,7 @@ def cs_fetch(path: str, data_selector: str, col_name: str) -> pd.DataFrame: - response = HTTP.get(f'https://coinank.com/indicatorapi/{path}') + response = HTTP.get(f'https://api.coinank.com/indicatorapi/{path}') response.raise_for_status() data = response.json()['data'] @@ -15,12 +15,10 @@ def cs_fetch(path: str, data_selector: str, col_name: str) -> pd.DataFrame: data_y = data[data_selector] assert len(data_x) == len(data_y), f'{len(data_x)=} != {len(data_y)=}' - df = pd.DataFrame( - { - 'Date': data_x[: len(data_y)], - col_name: data_y, - } - ) + df = pd.DataFrame({ + 'Date': data_x[: len(data_y)], + col_name: data_y, + }) df['Date'] = pd.to_datetime(df['Date'], unit='ms').dt.tz_localize(None) diff --git a/fetch_bitcoin_data.py b/fetch_bitcoin_data.py index c226f88..24316da 100644 --- a/fetch_bitcoin_data.py +++ b/fetch_bitcoin_data.py @@ -1,43 +1,121 @@ +from itertools import count, pairwise + import numpy as np import pandas as pd from filecache import filecache from utils import HTTP, mark_days_since, mark_highs_lows +HALVING_INTERVAL = 210_000 +GENESIS_BLOCK_REWARD = 50.0 +BLOCKS_PER_DAY = 144 -@filecache(7200) # 2 hours -def fetch_bitcoin_data() -> pd.DataFrame: + +def fetch_block_halving(): """ - Fetches historical Bitcoin data into a DataFrame. - Very early data is discarded due to high volatility. + Fetch Bitcoin halving data by probing raw blocks at halving heights. + """ + halving_data: list[tuple[pd.Timestamp, int, float]] = [] + + for halving_index in count(): + block_height = halving_index * HALVING_INTERVAL + block_reward = GENESIS_BLOCK_REWARD / (2**halving_index) + + response = HTTP.get(f'https://blockchain.info/rawblock/{block_height}') + if response.status_code == 404: + break + + response.raise_for_status() + block_time = response.json()['time'] + block_date = pd.to_datetime(block_time, unit='s').tz_localize(None).floor('d') + halving_data.append((block_date, block_height, block_reward)) + + return halving_data + + +def fetch_blockchain_data() -> pd.DataFrame: + """ + Fetches historical Bitcoin blockchain data from Blockchain.com API. + Uses miners-revenue chart for USD mining revenue. + Block heights and BTC generation are calculated from halving schedule. Returns: - DataFrame containing Bitcoin data. + DataFrame with Date, TotalBlocks, MinBlockID, MaxBlockID, + TotalGeneration, TotalGenerationUSD columns. """ - print('📈 Requesting historical Bitcoin data…') + halving_data = fetch_block_halving() + # Fetch mining revenue from Blockchain.com response = HTTP.get( - 'https://api.blockchair.com/bitcoin/blocks', + 'https://api.blockchain.info/charts/miners-revenue', params={ - 'a': 'date,count(),min(id),max(id),sum(generation),sum(generation_usd)', - 's': 'date(desc)', + 'timespan': 'all', + 'format': 'json', + 'sampled': 'false', }, ) + + # Create DataFrame from mining revenue data response.raise_for_status() - response_json = response.json() + df = pd.DataFrame(response.json()['values']) + df.columns = ['DateTimestamp', 'TotalGenerationUSD'] + df['Date'] = pd.to_datetime(df['DateTimestamp'], unit='s').dt.floor('d') + + # Calculate approximate block height for each day using linear interpolation + # between known halving points + def estimate_block_height(date: pd.Timestamp): + # Find the halving period this date falls into + for (start_date, start_height, _), (end_date, end_height, _) in pairwise(halving_data): + if start_date <= date < end_date: + # Linear interpolation within this halving period + total_days = (end_date - start_date).days + days_elapsed = (date - start_date).days + height = start_height + (end_height - start_height) * days_elapsed / total_days + return int(height) + + # After the last known halving, extrapolate + (last_date, last_height, _) = halving_data[-1] + days_since = (date - last_date).days + return int(last_height + days_since * BLOCKS_PER_DAY) + + def get_block_reward(block_height): + """Get block reward for a given block height.""" + halvings = block_height // HALVING_INTERVAL + return GENESIS_BLOCK_REWARD / (2**halvings) + + # Calculate block data for each day + df['MaxBlockID'] = df['Date'].apply(estimate_block_height) + df['MinBlockID'] = df['MaxBlockID'].shift(1, fill_value=0) + df['TotalBlocks'] = df['MaxBlockID'] - df['MinBlockID'] + + # Calculate BTC generation based on block reward + df['BlockReward'] = df['MaxBlockID'].apply(get_block_reward) + df['TotalGeneration'] = df['TotalBlocks'] * df['BlockReward'] * 1e8 # Convert to satoshis + + # Select and order columns to match original format + df = df[['Date', 'TotalBlocks', 'MinBlockID', 'MaxBlockID', 'TotalGeneration', 'TotalGenerationUSD']] + df = df.sort_values('Date').reset_index(drop=True) + + # Add halving markers + df['Halving'] = 0 + for _, block_height, _ in halving_data[1:]: + df.loc[(df['MinBlockID'] < block_height) & (df['MaxBlockID'] >= block_height), 'Halving'] = 1 + + return df - df = pd.DataFrame(response_json['data'][::-1]) - df.rename( - columns={ - 'date': 'Date', - 'count()': 'TotalBlocks', - 'min(id)': 'MinBlockID', - 'max(id)': 'MaxBlockID', - 'sum(generation)': 'TotalGeneration', - 'sum(generation_usd)': 'TotalGenerationUSD', - }, - inplace=True, - ) + +@filecache(7200) # 2 hours +def fetch_bitcoin_data() -> pd.DataFrame: + """ + Fetches historical Bitcoin data into a DataFrame. + Very early data is discarded due to high volatility. + + Returns: + DataFrame containing Bitcoin data. + """ + print('📈 Requesting historical Bitcoin data…') + + df = fetch_blockchain_data() df['Date'] = pd.to_datetime(df['Date']) df['TotalGeneration'] /= 1e8 @@ -57,7 +135,6 @@ def fetch_bitcoin_data() -> pd.DataFrame: df.reset_index(drop=True, inplace=True) df = fix_current_day_data(df) - df = add_block_halving_data(df) df = mark_highs_lows(df, 'Price', False, round(365 * 2), 180) # move 2021' peak to the first price peak @@ -82,12 +159,10 @@ def fetch_price_data() -> pd.DataFrame: response_x = [float(k) for k in response_json['data']['points']] response_y = [value['v'][0] for value in response_json['data']['points'].values()] - df = pd.DataFrame( - { - 'Date': response_x, - 'Price': response_y, - } - ) + df = pd.DataFrame({ + 'Date': response_x, + 'Price': response_y, + }) df['Date'] = pd.to_datetime(df['Date'], unit='s').dt.tz_localize(None).dt.floor('d') df.sort_values(by='Date', inplace=True) df.drop_duplicates('Date', keep='last', inplace=True) @@ -98,41 +173,10 @@ def fetch_price_data() -> pd.DataFrame: def fix_current_day_data(df: pd.DataFrame) -> pd.DataFrame: row = df.iloc[-1].copy() - target_total_blocks = 24 * 6 - target_scale = target_total_blocks / row['TotalBlocks'] + target_scale = BLOCKS_PER_DAY / row['TotalBlocks'] for col_name in ['TotalBlocks', 'TotalGeneration', 'TotalGenerationUSD']: row[col_name] *= target_scale df.iloc[-1] = row return df - - -def add_block_halving_data(df: pd.DataFrame) -> pd.DataFrame: - reward_halving_every = 210000 - current_block_halving_id = reward_halving_every - current_block_production = 50 - df['Halving'] = 0 - df['NextHalvingBlock'] = current_block_halving_id - - while True: - df.loc[ - (current_block_halving_id - reward_halving_every) <= df['MaxBlockID'], - 'BlockGeneration', - ] = current_block_production - - block_halving_row = df[ - (df['MinBlockID'] <= current_block_halving_id) & (df['MaxBlockID'] >= current_block_halving_id) - ].squeeze() - - if block_halving_row.shape[0] == 0: - break - - current_block_halving_id += reward_halving_every - current_block_production /= 2 - df.loc[block_halving_row.name, 'Halving'] = 1 - df.loc[df.index > block_halving_row.name, 'NextHalvingBlock'] = current_block_halving_id - - df['DaysToHalving'] = pd.to_timedelta((df['NextHalvingBlock'] - df['MaxBlockID']) / (24 * 6), unit='D') - df['NextHalvingDate'] = df['Date'] + df['DaysToHalving'] - return df diff --git a/metrics/pi_cycle.py b/metrics/pi_cycle.py index 424f451..24a6dfc 100644 --- a/metrics/pi_cycle.py +++ b/metrics/pi_cycle.py @@ -63,4 +63,8 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: sns.lineplot(data=df, x='Date', y='PiCycleIndexNoNa', ax=ax[0]) add_common_markers(df, ax[0]) + sns.lineplot(data=df, x='Date', y='PiCycleDiff', ax=ax[1]) + sns.lineplot(data=df, x='Date', y='PiCycleDiffThreshold', ax=ax[1], linestyle='--') + add_common_markers(df, ax[1], price_line=False) + return df['PiCycleIndex'] diff --git a/metrics/puell_multiple.py b/metrics/puell_multiple.py index 50d31b9..75cffca 100644 --- a/metrics/puell_multiple.py +++ b/metrics/puell_multiple.py @@ -4,9 +4,8 @@ from matplotlib.axes import Axes from sklearn.linear_model import LinearRegression -from api.coinsoto_api import cs_fetch from metrics.base_metric import BaseMetric -from utils import add_common_markers +from utils import add_common_markers, mark_highs_lows class PuellMetric(BaseMetric): @@ -19,19 +18,16 @@ def description(self) -> str: return 'Puell Multiple' def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: - df = df.merge( - cs_fetch( - path='getPuellMultiple', - data_selector='puellMultiplList', - col_name='Puell', - ), - on='Date', - how='left', - ) + # Calculate Puell Multiple locally from mining revenue data + # Puell = daily_mining_revenue / 365-day_MA_of_mining_revenue + # TotalGenerationUSD contains daily mining revenue in USD from Blockchain.com + df['Puell'] = df['TotalGenerationUSD'] / df['TotalGenerationUSD'].rolling(window=365, min_periods=1).mean() df['Puell'] = df['Puell'].ffill() df['PuellLog'] = np.log(df['Puell']) - high_rows = df.loc[df['PriceHigh'] == 1] + df = mark_highs_lows(df, 'PuellLog', True, round(365 * 2), 365) + high_rows = df.loc[(df['PuellLogHigh'] == 1) & (df.index > 365)] + high_x = high_rows.index.values.reshape(-1, 1) high_y = high_rows['PuellLog'].values.reshape(-1, 1) @@ -43,7 +39,9 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: lin_model = LinearRegression() lin_model.fit(high_x, high_y) - df['PuellLogHighModel'] = lin_model.predict(x) + predictions = lin_model.predict(x) + min_peak = high_y.min() + df['PuellLogHighModel'] = np.maximum(predictions, min_peak) # lin_model.fit(low_x, low_y) # df['PuellLogLowModel'] = lin_model.predict(x) diff --git a/metrics/two_year_moving_average.py b/metrics/two_year_moving_average.py index eb002fd..ec225f4 100644 --- a/metrics/two_year_moving_average.py +++ b/metrics/two_year_moving_average.py @@ -4,7 +4,6 @@ from matplotlib.axes import Axes from sklearn.linear_model import LinearRegression -from api.coinsoto_api import cs_fetch from metrics.base_metric import BaseMetric from utils import add_common_markers @@ -19,15 +18,9 @@ def description(self) -> str: return '2 Year Moving Average' def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series: - df = df.merge( - cs_fetch( - path='getBtcMultiplier', - data_selector='mA730List', - col_name='2YMA', - ), - on='Date', - how='left', - ) + # Calculate 2-year (730-day) moving average locally from price data + # No external API needed - we have the price data already + df['2YMA'] = df['Price'].rolling(window=730, min_periods=1).mean() df['2YMA'] = df['2YMA'].ffill() df['2YMALog'] = np.log(df['2YMA']) df['2YMALogDiff'] = df['PriceLog'] - df['2YMALog'] diff --git a/utils.py b/utils.py index fce7efa..e19b85d 100644 --- a/utils.py +++ b/utils.py @@ -237,4 +237,4 @@ async def send_error_notification(exception: Exception) -> bool: f'
{"".join(traceback.format_exception(exception))}
', parse_mode='HTML', ) - return True + return True \ No newline at end of file