diff --git a/scripts_staging/Backend/Uptime Kuma Monitoring For Tactical.py b/scripts_staging/Backend/Uptime Kuma Monitoring For Tactical.py index e8938e2f..ff6cb794 100644 --- a/scripts_staging/Backend/Uptime Kuma Monitoring For Tactical.py +++ b/scripts_staging/Backend/Uptime Kuma Monitoring For Tactical.py @@ -1,542 +1,524 @@ -#!/usr/bin/python3 -#public -''' -.SYNOPSIS - Python script designed to automatically update the interface of Uptime-Kuma based online machines for Tactical. - -.DESCRIPTION - This script operates in two parts. The first part retrieves information from the field and the Agent ID from the Tactical Swagger - After fetching the information, it checks whether the websites still exist in Tactical. If they don't, the script removes them from the dashboard. - Additionally, it verifies if the sites are already present; if not, it creates them, specifying the name, URL, and Agent ID in the description. - -.ADDITIONAL INFORMATIONS - API : https://uptime-kuma-api.readthedocs.io/en/latest/index.html - Docker-Compose : uptime-kuma on dockge - Version : 1.5.2 - -.NOTE - Author: MSA/SAN - Date: 17.08.24 - -.EXEMPLE -endpoint_uptimekuma=UPTIME URL -user_uptimekuma=UPTIME USER -password_uptimekuma={{global.uptimepassword}} -rmm_key_for_uptime={{global.rmm_key_for_uptime_script}} -rmm_url=https://RMM API URL/agents -CustomFieldID=11111111 - -.TODO - When a hostname is removed/moved, this script doesn't automatically delete it. Need to be fix. - The HTTP protocol is automatically replaced by HTTPS. This should be adjusted to retain HTTP when specific keywords are used. - Remove the URL from the display name. -''' - - -# Import standard modules -import sys -import subprocess -import re -import os -import requests -import time - -# Function to install missing packages -def install(package): - subprocess.check_call([sys.executable, "-m", "pip", "install", package]) - -# Attempt to import the 'uptime_kuma_api' module -try: - import uptime_kuma_api -except ImportError: - print("Module 'uptime_kuma_api' not found. Installing...") - install("uptime_kuma_api") - -# Import additional modules needed for interacting with Uptime-Kuma API -from uptime_kuma_api import UptimeKumaApi, MonitorType - -# Initialise connection to the Uptime-Kuma API -api = UptimeKumaApi(os.environ.get('endpoint_uptimekuma')) -api.login(os.environ.get('user_uptimekuma'), os.environ.get('password_uptimekuma')) - -# Define API key and URL from environment variables -api_key = os.getenv('rmm_key_for_uptime') -url = os.getenv('rmm_url') -custom_field_id = int(os.getenv('CustomFieldID')) - -# Define headers for the API request -headers = { - "X-API-KEY": api_key, - "Accept": "application/json" -} - -try: - # Send a GET request to the specified URL - response = requests.get(url, headers=headers) - - if response.status_code == 200: - # Parse the JSON response - data = response.json() - - if isinstance(data, list): - for agent in data: - # Check if 'custom_fields' is present in the agent data - if 'custom_fields' in agent: - # Extract values from custom fields where the field ID is custom_field_id - filtered_values = [cf['value'] for cf in agent['custom_fields'] if cf.get('field') == custom_field_id and cf.get('value')] - - # Process agents with at least one relevant custom field - if filtered_values: - - # Extract agent details - agent_id = agent.get('agent_id', 'N/A') - default_hostname = agent.get('hostname', 'N/A') - site_name = agent.get('site_name', 'N/A') - client_name = agent.get('client_name', 'N/A') - public_ip = agent.get('public_ip', 'N/A') - - # Get 5 first character - agent_id_5_char = agent_id[:5] - - # Hostname full name - hostname = f"{default_hostname} [{agent_id_5_char}]" - - # Space in order to have an output more clearly - print() - - # Check and deploy client monitor - monitors = api.get_monitors() - client_monitor = next((monitor for monitor in monitors if monitor.get('name') == client_name), None) - - if client_monitor: - print(f"{client_name} already exists") - else: - api.add_monitor( - type=MonitorType.GROUP, - name=client_name, - description="Client" - ) - print(f"Client {client_name} has been created") - - # Check and deploy site monitor under the client - monitors = api.get_monitors() - client_monitor = next((monitor for monitor in monitors if monitor.get('name') == client_name), None) - - if any(monitor.get('name') == site_name and monitor.get('parent') == client_monitor.get('id') for monitor in monitors): - print(f"{site_name} already exists on {client_name}") - else: - api.add_monitor( - type=MonitorType.GROUP, - name=site_name, - parent=client_monitor.get('id'), - description="Site" - ) - print(f"Site {site_name} has been created on {client_name}") - - # Check and deploy hostname monitor under the site - monitors = api.get_monitors() - site_monitor = next((monitor for monitor in monitors if monitor.get('name') == site_name and monitor.get('parent') == client_monitor.get('id')), None) - - if site_monitor: - if any(monitor.get('name') == hostname and monitor.get('parent') == site_monitor.get('id') for monitor in monitors): - print(f"{hostname} already exists on {client_name} / {site_name}") - else: - api.add_monitor( - type=MonitorType.GROUP, - name=hostname, - parent=site_monitor.get('id'), - description="Hostname" - ) - print(f"Hostname {hostname} - {agent_id} has been created on {client_name} / {site_name}") - - # Space in order to have an output more clearly - print() - - # Add specific monitors based on filtered values - monitors = api.get_monitors() - monitor_id = None - - # Find monitor ID for hostname - for monitor in monitors: - if monitor.get('name') == hostname: - monitor_id = monitor.get('id') - - # Get relevant monitors that are children of the hostname monitor - relevant_monitors = [monitor for monitor in monitors if monitor.get('parent') == monitor_id] - - for value in filtered_values: - - # Add TCP port monitors with IP addresses - tcp_ports_with_ip_matches = re.findall(r'(\d+):(\d+\.\d+\.\d+\.\d+)', value) - - for port, ip in tcp_ports_with_ip_matches: - if port.isdigit(): - port_int = int(port) - - monitor_name = f"{port_int} - {ip} [{agent_id_5_char}]" - - if any(monitor.get('name') == monitor_name for monitor in relevant_monitors): - print(f"{monitor_name} already exists on {client_name} / {site_name} / {hostname}") - else: - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - port=port_int, - interval=60, - retryInterval=20, - maxretries=20, - parent=monitor_id, - description=f"Agent ID: {agent_id}", - hostname=ip - ) - print(f"Monitoring TCP for {monitor_name} has been created on {client_name} / {site_name} / {hostname}") - - # Add TCP port monitors with default IP addresses - value = re.sub(r'\d+:\d+\.\d+\.\d+\.\d+', '', value) - tcp_ports_no_ip_matches = re.findall(r'\b\d{1,5}\b', value) - - for port in tcp_ports_no_ip_matches: - if port.isdigit(): - port_int = int(port) - - monitor_name = f"{port_int} - {public_ip} [{agent_id_5_char}]" - - if any(monitor.get('name') == monitor_name for monitor in relevant_monitors): - print(f"{monitor_name} already exists on {client_name} / {site_name} / {hostname}") - else: - api.add_monitor( - type=MonitorType.PORT, - name=monitor_name, - port=port_int, - interval=60, - retryInterval=20, - maxretries=20, - parent=monitor_id, - description=f"Agent ID: {agent_id}", - hostname=public_ip - ) - print(f"Monitoring TCP for {monitor_name} has been created on {client_name} / {site_name} / {hostname}") - - # Add HTTP monitors for URLs - http_section_match = re.search(r'HTTP:\s*((?:(?!TCP:|KEYWORD:)[\s\S])*)', value) - if http_section_match: - http_section = http_section_match.group(1).strip() - http_urls = [url.strip() for url in http_section.split('\n') if url.strip()] - - for url in http_urls: - original_url = url - - url = re.sub(r'^(https?:\/\/)+', '', url) - url = re.sub(r'^\/+', '', url) - url = re.sub(r'\s+', ' ', url) - url = url.strip() - - if original_url.lower().startswith('http:'): - protocol = 'http://' - elif original_url.lower().startswith('https:'): - protocol = 'https://' - else: - protocol = 'https://' - - full_url = f"{protocol}{url}" - monitor_name = f"{full_url} [{agent_id_5_char}]" - - if re.match(r'^https?:\/\/[a-zA-Z0-9-._~:/?#\[\]@!$&\'()*+,;%=]+$', full_url): - if any(monitor.get('name') == monitor_name for monitor in relevant_monitors): - print(f"{monitor_name} already exists on {client_name} / {site_name} / {hostname}") - else: - api.add_monitor( - type=MonitorType.HTTP, - name=monitor_name, - url=full_url, - interval=60, - retryInterval=20, - maxretries=20, - timeout=15, - expiryNotification=True, - parent=monitor_id, - description=f"Agent ID: {agent_id}", - hostname=public_ip - ) - print(f"Monitoring HTTP for {monitor_name} has been created on {client_name} / {site_name} / {hostname}") - else: - print(f"Invalid HTTP URL: {full_url}") - - # Add KEYWORD monitors for keyword-based URLs - keyword_urls_matches = re.findall(r'KEYWORD:\s*((?:[^\n]+(?:\n(?!TCP:|HTTP:))?)+)', value, re.DOTALL) - if keyword_urls_matches: - for match in keyword_urls_matches: - urls = match.strip().split('\n') - for url in urls: - url = url.strip() - - if ':' in url: - base_url, keyword = url.rsplit(':', 1) - else: - base_url = url - keyword = 'test' - - original_protocol = '' - if base_url.lower().startswith('http://'): - original_protocol = 'http://' - elif base_url.lower().startswith('https://'): - original_protocol = 'https://' - - base_url = re.sub(r'^(https?:\/\/)+', '', base_url) - base_url = re.sub(r'^\/+', '', base_url) - base_url = re.sub(r'\s+', ' ', base_url) - base_url = base_url.strip() - - if original_protocol: - base_url = f"{original_protocol}{base_url}" - elif not base_url.startswith(('http://', 'https://')): - base_url = f"https://{base_url}" - - keyword = keyword.strip() - monitor_name = f"{base_url} - {keyword} [{agent_id_5_char}]" - if any(monitor.get('name') == monitor_name for monitor in relevant_monitors): - print(f"{monitor_name} already exists on {client_name} / {site_name} / {hostname}") - else: - api.add_monitor( - type=MonitorType.KEYWORD, - name=monitor_name, - url=base_url, - keyword=keyword, - interval=60, - retryInterval=20, - maxretries=20, - timeout=15, - expiryNotification=True, - parent=monitor_id, - description=f"Agent ID: {agent_id}", - hostname=public_ip - ) - print(f"Monitoring KEYWORD for {monitor_name} has been created on {client_name} / {site_name} / {hostname}") - - # Space in order to have an output more clearly - print() - - # Reset default values - monitors = api.get_monitors() - monitor_id = None - - # Find monitor ID for hostname - for monitor in monitors: - if monitor.get('name') == hostname: - monitor_id = monitor.get('id') - - # Get relevant monitors that are children of the hostname monitor - relevant_monitors = [monitor for monitor in monitors if monitor.get('parent') == monitor_id] - - # Check and remove TCP port monitors with default IP if no longer relevant - for monitor in relevant_monitors: - if monitor.get('type') == 'port': - monitor_name = monitor.get('name') - monitor_id = monitor.get('id') - exists_in_value = False - - # Extract port, IP, and agent_id from monitor name - port_ip_match = re.match(r'(\d+) - ([\d\.]+) \[(.*?)\]', monitor_name) - if port_ip_match: - monitor_port, monitor_ip, monitor_agent_id = port_ip_match.groups() - - for value in filtered_values: - # Check for ports with specific IP - if re.search(rf'{monitor_port}:{monitor_ip}', value): - exists_in_value = True - break - - # Check for ports with public IP - if monitor_ip == public_ip: - tcp_ports_no_ip = re.sub(r'\d+:\d+\.\d+\.\d+\.\d+', '', value) - tcp_ports = re.findall(r'\b(\d+)\b', tcp_ports_no_ip) - if monitor_port in tcp_ports: - exists_in_value = True - break - - # Check if the monitor belongs to the current agent - if monitor_agent_id != agent_id_5_char: - exists_in_value = True # Don't delete monitors from other agents - - if not exists_in_value: - print(f"{monitor_name} does not exist anymore on the agent and has been deleted on {client_name} / {site_name} / {hostname}") - api.delete_monitor(monitor_id) - - # Check and remove HTTP monitors if no longer relevant - for monitor in relevant_monitors: - if monitor.get('type') == 'http': - monitor_name = monitor.get('name') - monitor_id = monitor.get('id') - exists_in_value = False - - for value in filtered_values: - http_urls_matches = re.findall(r'HTTP:\s*((?:[^\n]+\n?)+)', value, re.DOTALL) - if http_urls_matches: - for match in http_urls_matches: - urls = match.strip().split('\n') - for url in urls: - url = url.strip() - - url = re.sub(r'^(https?:\/\/)+', '', url) - url = re.sub(r'^\/+', '', url) - url = re.sub(r'\s+', ' ', url) - url = url.strip() - - if url.lower().startswith('https:'): - protocol = 'https://' - url = url[6:] - elif url.lower().startswith('http:'): - protocol = 'http://' - url = url[5:] - else: - protocol = 'https://' - - full_url = f"{protocol}{url}" - expected_name = f"{full_url} [{agent_id_5_char}]" - - if monitor_name == expected_name: - exists_in_value = True - break - if exists_in_value: - break - if exists_in_value: - break - if exists_in_value: - break - - if not exists_in_value: - print(f"{monitor_name} does not exist anymore on the agent and has been deleted on {client_name} / {site_name} / {hostname}") - api.delete_monitor(monitor_id) - - # Check and remove KEYWORD monitors if no longer relevant - for monitor in relevant_monitors: - if monitor.get('type') == 'keyword': - monitor_name = monitor.get('name') - monitor_id = monitor.get('id') - exists_in_value = False - - for value in filtered_values: - keyword_urls_matches = re.findall(r'KEYWORD:\s*((?:[^\n]+\n?)+)', value, re.DOTALL) - if keyword_urls_matches: - for match in keyword_urls_matches: - urls = match.strip().split('\n') - for url in urls: - url = url.strip() - - if ':' in url: - base_url, keyword = url.rsplit(':', 1) - else: - base_url = url - keyword = 'test' - - base_url = re.sub(r'^(https?:\/\/)+', '', base_url) - base_url = re.sub(r'^\/+', '', base_url) - base_url = re.sub(r'\s+', ' ', base_url) - base_url = base_url.strip() - - if not base_url.startswith(('http://', 'https://')): - base_url = f"https://{base_url}" - - keyword = keyword.strip() - - expected_name = f"{base_url} - {keyword} [{agent_id_5_char}]" - if monitor_name == expected_name: - exists_in_value = True - break - if exists_in_value: - break - if exists_in_value: - break - if exists_in_value: - break - - if not exists_in_value: - print(f"{monitor_name} does not exist anymore on the agent and has been deleted on {client_name} / {site_name} / {hostname}") - api.delete_monitor(monitor_id) - - # Space in order to have an output more clearly - print() - - # Additional wait to ensure API is fully synced - time.sleep(1) - - - # Get custom fields with the field ID from the environment variable that have no value for the given agent - empty_values = [cf for cf in agent['custom_fields'] if cf.get('field') == custom_field_id and not cf.get('value')] - - # Proceed only if there are custom fields that are empty - if empty_values: - # Fetch agent details with fallback defaults if values are missing - # Extract agent details - agent_id = agent.get('agent_id', 'N/A') - default_hostname = agent.get('hostname', 'N/A') - site_name = agent.get('site_name', 'N/A') - client_name = agent.get('client_name', 'N/A') - - # Get 5 first character - agent_id_5_char = agent_id[:5] - - # Hostname full name - hostname = f"{default_hostname} [{agent_id_5_char}]" - - # Get the list of all relevant monitors via API - relevant_monitors = api.get_monitors() - - # Loop through each monitor to check for matches with the agent's hostname - for monitor in relevant_monitors: - monitor_name = monitor.get('name') - monitor_id = monitor.get('id') - monitor_child = monitor.get('childrenIDs', []) - - # If the monitor's name matches the agent's hostname, proceed - if monitor_name == hostname: - - # Loop through child monitors of the matched monitor - for child in monitor_child: - # Get the name of the child monitor, with a fallback to 'Unknown' if not found - child_monitor_name = api.get_monitor(child).get('name', 'Unknown') - - # Log a message about the child monitor being deleted - print(f"{child_monitor_name} does not exist anymore on the agent and has been deleted on {client_name} / {site_name} / {hostname}") - - # Delete the child monitor via API - api.delete_monitor(child) - - # Additional wait to ensure API is fully synced - time.sleep(1) - - # Check and remove group monitors with no children - NoSubMonitor = True - - while NoSubMonitor: - relevant_monitors = api.get_monitors() - NoSubMonitor = False - - for monitor in relevant_monitors: - if monitor.get('type') == 'group': - monitor_name = monitor.get('name') - monitor_id = monitor.get('id') - children_ids = monitor.get('childrenIDs', []) - - if not children_ids: - print(f"{monitor_name} does not have any children") - api.delete_monitor(monitor_id) - NoSubMonitor = True # Continue loop if any monitor was deleted - - # Additional wait to ensure API is fully synced - time.sleep(1) - - time.sleep(2) # Sleep to avoid rapid API calls - - else: - print("Unexpected data format received.") - - else: - print(f"Request failed. Status code: {response.status_code}") - print(f"Error message: {response.text}") - -except requests.exceptions.RequestException as e: - print(f"An error occurred during the request: {e}") - -# Disconnect from the API service -api.disconnect() \ No newline at end of file +#!/usr/bin/python3 +''' +.SYNOPSIS + Python script designed to automatically update the interface of Uptime-Kuma based online machines for Tactical. + +.DESCRIPTION + This script operates in two parts. The first part retrieves information from the field and the Agent ID from the Tactical Swagger + After fetching the information, it checks whether the websites still exist in Tactical. If they don't, the script removes them from the dashboard. + Additionally, it verifies if the sites are already present; if not, it creates them, specifying the name, URL, and Agent ID in the description. + +.NOTE + Author: MSA/SAN + Date: 17.08.24 + API : https://uptime-kuma-api.readthedocs.io/en/latest/index.html + #PUBLIC + +.EXEMPLE + endpoint_uptimekuma=UPTIME URL + user_uptimekuma=UPTIME USER + password_uptimekuma={{global.uptimepassword}} + rmm_key_for_uptime={{global.rmm_key_for_uptime_script}} + rmm_url=https://RMM API URL/agents + CustomFieldID=11111111 + +.CHANGELOG + 02.06.26 SAN Big code cleanup, multiple timeouts fixes, https cleanup, value mutation bug, removed redundant calls, added startup check, modularisation + +.TODO + Alternative to the API used will be needed as it is unmaintained and does not look compatible with uptime V2 + When a hostname is removed/moved, this script doesn't automatically delete it. Need to be fix. + The HTTP protocol is automatically replaced by HTTPS. This should be adjusted to retain HTTP when specific keywords are used. + Remove the URL from the display name. + +''' + +import sys +import subprocess +import re +import os +import requests +import time +import socketio + + +def install(package): + subprocess.check_call([sys.executable, "-m", "pip", "install", package]) + + +try: + import uptime_kuma_api +except ImportError: + print("Module 'uptime_kuma_api' not found. Installing...") + install("uptime_kuma_api") + +from uptime_kuma_api import UptimeKumaApi, MonitorType + + +def safe_api_call(fn, *args, retries=3, delay=2, **kwargs): + for attempt in range(retries): + try: + return fn(*args, **kwargs) + except socketio.exceptions.TimeoutError: + if attempt == retries - 1: + print(f"TimeoutError after {retries} retries") + return None + print(f"Timeout, retrying ({attempt + 1}/{retries})...") + time.sleep(delay) + except Exception as e: + if attempt == retries - 1: + print(f"Error after {retries} retries: {e}") + return None + print(f"Error (attempt {attempt + 1}/{retries}): {e}") + time.sleep(delay) + + +def normalize_url(raw_url): + url = re.sub(r'^(https?:\/\/)+', '', raw_url) + url = re.sub(r'^\/+', '', url) + url = re.sub(r'\s+', ' ', url) + url = url.strip() + if raw_url.lower().startswith('http:'): + protocol = 'http://' + elif raw_url.lower().startswith('https:'): + protocol = 'https://' + else: + protocol = 'https://' + return f"{protocol}{url}" + + +def get_hostname_monitor_id(monitors, hostname): + for monitor in monitors: + if monitor.get('name') == hostname: + return monitor.get('id') + return None + + +def validate_env_vars(): + required = [ + 'endpoint_uptimekuma', 'user_uptimekuma', 'password_uptimekuma', + 'rmm_key_for_uptime', 'rmm_url', 'CustomFieldID' + ] + missing = [v for v in required if not os.environ.get(v)] + if missing: + print(f"Missing required environment variables: {', '.join(missing)}") + sys.exit(1) + + +if __name__ == '__main__': + validate_env_vars() + + endpoint = os.environ.get('endpoint_uptimekuma') + user = os.environ.get('user_uptimekuma') + password = os.environ.get('password_uptimekuma') + + api = UptimeKumaApi(endpoint) + api.timeout = 30 + safe_api_call(api.login, user, password) + + api_key = os.getenv('rmm_key_for_uptime') + url = os.getenv('rmm_url') + custom_field_id = int(os.getenv('CustomFieldID')) + + headers = { + "X-API-KEY": api_key, + "Accept": "application/json" + } + + try: + response = requests.get(url, headers=headers) + + if response.status_code == 200: + data = response.json() + + if isinstance(data, list): + for agent in data: + if 'custom_fields' in agent: + filtered_values = [ + cf['value'] for cf in agent['custom_fields'] + if cf.get('field') == custom_field_id and cf.get('value') + ] + + if filtered_values: + agent_id = agent.get('agent_id', 'N/A') + default_hostname = agent.get('hostname', 'N/A') + site_name = agent.get('site_name', 'N/A') + client_name = agent.get('client_name', 'N/A') + public_ip = agent.get('public_ip', 'N/A') + + agent_id_5_char = agent_id[:5] + hostname = f"{default_hostname} [{agent_id_5_char}]" + + print() + + monitors = api.get_monitors() + + client_monitor = next( + (m for m in monitors if m.get('name') == client_name), None + ) + + if client_monitor: + print(f"{client_name} already exists") + else: + safe_api_call( + api.add_monitor, + type=MonitorType.GROUP, + name=client_name, + description="Client" + ) + print(f"Client {client_name} has been created") + monitors = api.get_monitors() + client_monitor = next( + (m for m in monitors if m.get('name') == client_name), None + ) + + if any( + m.get('name') == site_name and m.get('parent') == client_monitor.get('id') + for m in monitors + ): + print(f"{site_name} already exists on {client_name}") + else: + safe_api_call( + api.add_monitor, + type=MonitorType.GROUP, + name=site_name, + parent=client_monitor.get('id'), + description="Site" + ) + print(f"Site {site_name} has been created on {client_name}") + + monitors = api.get_monitors() + site_monitor = next( + (m for m in monitors + if m.get('name') == site_name and m.get('parent') == client_monitor.get('id')), + None + ) + + if site_monitor: + if any( + m.get('name') == hostname and m.get('parent') == site_monitor.get('id') + for m in monitors + ): + print(f"{hostname} already exists on {client_name} / {site_name}") + else: + safe_api_call( + api.add_monitor, + type=MonitorType.GROUP, + name=hostname, + parent=site_monitor.get('id'), + description="Hostname" + ) + print(f"Hostname {hostname} - {agent_id} has been created on {client_name} / {site_name}") + + print() + + monitors = api.get_monitors() + hostname_monitor_id = get_hostname_monitor_id(monitors, hostname) + relevant_monitors = [ + m for m in monitors if m.get('parent') == hostname_monitor_id + ] + + for value in filtered_values: + tcp_ports_with_ip_matches = re.findall(r'(\d+):(\d+\.\d+\.\d+\.\d+)', value) + + for port, ip in tcp_ports_with_ip_matches: + port_int = int(port) + monitor_name = f"{port_int} - {ip} [{agent_id_5_char}]" + + if any(m.get('name') == monitor_name for m in relevant_monitors): + print(f"{monitor_name} already exists on {client_name} / {site_name} / {hostname}") + else: + safe_api_call( + api.add_monitor, + type=MonitorType.PORT, + name=monitor_name, + port=port_int, + interval=60, + retryInterval=20, + maxretries=20, + parent=hostname_monitor_id, + description=f"Agent ID: {agent_id}", + hostname=ip + ) + print(f"Monitoring TCP for {monitor_name} has been created on {client_name} / {site_name} / {hostname}") + + cleaned_value = re.sub(r'\d+:\d+\.\d+\.\d+\.\d+', '', value) + tcp_ports_no_ip_matches = re.findall(r'\b\d{1,5}\b', cleaned_value) + + for port in tcp_ports_no_ip_matches: + port_int = int(port) + monitor_name = f"{port_int} - {public_ip} [{agent_id_5_char}]" + + if any(m.get('name') == monitor_name for m in relevant_monitors): + print(f"{monitor_name} already exists on {client_name} / {site_name} / {hostname}") + else: + safe_api_call( + api.add_monitor, + type=MonitorType.PORT, + name=monitor_name, + port=port_int, + interval=60, + retryInterval=20, + maxretries=20, + parent=hostname_monitor_id, + description=f"Agent ID: {agent_id}", + hostname=public_ip + ) + print(f"Monitoring TCP for {monitor_name} has been created on {client_name} / {site_name} / {hostname}") + + http_section_match = re.search( + r'HTTP:\s*((?:(?!TCP:|KEYWORD:)[\s\S])*)', value + ) + if http_section_match: + http_section = http_section_match.group(1).strip() + http_urls = [u.strip() for u in http_section.split('\n') if u.strip()] + + for raw_url in http_urls: + full_url = normalize_url(raw_url) + monitor_name = f"{full_url} [{agent_id_5_char}]" + + if re.match(r'^https?:\/\/[a-zA-Z0-9-._~:/?#\[\]@!$&\'()*+,;%=]+$', full_url): + if any(m.get('name') == monitor_name for m in relevant_monitors): + print(f"{monitor_name} already exists on {client_name} / {site_name} / {hostname}") + else: + safe_api_call( + api.add_monitor, + type=MonitorType.HTTP, + name=monitor_name, + url=full_url, + interval=60, + retryInterval=20, + maxretries=20, + timeout=15, + expiryNotification=True, + parent=hostname_monitor_id, + description=f"Agent ID: {agent_id}", + hostname=public_ip + ) + print(f"Monitoring HTTP for {monitor_name} has been created on {client_name} / {site_name} / {hostname}") + else: + print(f"Invalid HTTP URL: {full_url}") + + keyword_urls_matches = re.findall( + r'KEYWORD:\s*((?:[^\n]+(?:\n(?!TCP:|HTTP:))?)+)', value, re.DOTALL + ) + if keyword_urls_matches: + for match in keyword_urls_matches: + urls = match.strip().split('\n') + for entry in urls: + entry = entry.strip() + if ':' in entry: + base_url_raw, keyword = entry.rsplit(':', 1) + else: + base_url_raw = entry + keyword = 'test' + + full_url = normalize_url(base_url_raw) + keyword = keyword.strip() + monitor_name = f"{full_url} - {keyword} [{agent_id_5_char}]" + + if any(m.get('name') == monitor_name for m in relevant_monitors): + print(f"{monitor_name} already exists on {client_name} / {site_name} / {hostname}") + else: + safe_api_call( + api.add_monitor, + type=MonitorType.KEYWORD, + name=monitor_name, + url=full_url, + keyword=keyword, + interval=60, + retryInterval=20, + maxretries=20, + timeout=15, + expiryNotification=True, + parent=hostname_monitor_id, + description=f"Agent ID: {agent_id}", + hostname=public_ip + ) + print(f"Monitoring KEYWORD for {monitor_name} has been created on {client_name} / {site_name} / {hostname}") + + print() + + monitors = api.get_monitors() + hostname_monitor_id = get_hostname_monitor_id(monitors, hostname) + relevant_monitors = [ + m for m in monitors if m.get('parent') == hostname_monitor_id + ] + + for monitor in relevant_monitors: + if monitor.get('type') == 'port': + monitor_name = monitor.get('name') + child_monitor_id = monitor.get('id') + exists_in_value = False + + port_ip_match = re.match( + r'(\d+) - ([\d\.]+) \[(.*?)\]', monitor_name + ) + if port_ip_match: + monitor_port, monitor_ip, monitor_agent_id = port_ip_match.groups() + + for value in filtered_values: + if re.search(rf'{monitor_port}:{monitor_ip}', value): + exists_in_value = True + break + + if monitor_ip == public_ip: + cleaned = re.sub( + r'\d+:\d+\.\d+\.\d+\.\d+', '', value + ) + tcp_ports = re.findall(r'\b(\d+)\b', cleaned) + if monitor_port in tcp_ports: + exists_in_value = True + break + + if monitor_agent_id != agent_id_5_char: + exists_in_value = True + + if not exists_in_value: + print( + f"{monitor_name} does not exist anymore on the agent " + f"and has been deleted on {client_name} / {site_name} / {hostname}" + ) + safe_api_call(api.delete_monitor, child_monitor_id) + + for monitor in relevant_monitors: + if monitor.get('type') == 'http': + monitor_name = monitor.get('name') + child_monitor_id = monitor.get('id') + exists_in_value = False + + for value in filtered_values: + http_urls_matches = re.findall( + r'HTTP:\s*((?:[^\n]+\n?)+)', value, re.DOTALL + ) + if http_urls_matches: + for match in http_urls_matches: + urls = match.strip().split('\n') + for raw_url in urls: + raw_url = raw_url.strip() + full_url = normalize_url(raw_url) + expected_name = f"{full_url} [{agent_id_5_char}]" + if monitor_name == expected_name: + exists_in_value = True + break + if exists_in_value: + break + if exists_in_value: + break + if exists_in_value: + break + + if not exists_in_value: + print( + f"{monitor_name} does not exist anymore on the agent " + f"and has been deleted on {client_name} / {site_name} / {hostname}" + ) + safe_api_call(api.delete_monitor, child_monitor_id) + + for monitor in relevant_monitors: + if monitor.get('type') == 'keyword': + monitor_name = monitor.get('name') + child_monitor_id = monitor.get('id') + exists_in_value = False + + for value in filtered_values: + keyword_urls_matches = re.findall( + r'KEYWORD:\s*((?:[^\n]+\n?)+)', value, re.DOTALL + ) + if keyword_urls_matches: + for match in keyword_urls_matches: + urls = match.strip().split('\n') + for entry in urls: + entry = entry.strip() + if ':' in entry: + base_url_raw, keyword = entry.rsplit(':', 1) + else: + base_url_raw = entry + keyword = 'test' + + full_url = normalize_url(base_url_raw) + keyword = keyword.strip() + expected_name = f"{full_url} - {keyword} [{agent_id_5_char}]" + if monitor_name == expected_name: + exists_in_value = True + break + if exists_in_value: + break + if exists_in_value: + break + if exists_in_value: + break + + if not exists_in_value: + print( + f"{monitor_name} does not exist anymore on the agent " + f"and has been deleted on {client_name} / {site_name} / {hostname}" + ) + safe_api_call(api.delete_monitor, child_monitor_id) + + print() + time.sleep(1) + + empty_values = [ + cf for cf in agent['custom_fields'] + if cf.get('field') == custom_field_id and not cf.get('value') + ] + + if empty_values: + agent_id = agent.get('agent_id', 'N/A') + default_hostname = agent.get('hostname', 'N/A') + site_name = agent.get('site_name', 'N/A') + client_name = agent.get('client_name', 'N/A') + + agent_id_5_char = agent_id[:5] + hostname = f"{default_hostname} [{agent_id_5_char}]" + + relevant_monitors = api.get_monitors() + + for monitor in relevant_monitors: + monitor_name = monitor.get('name') + monitor_id = monitor.get('id') + monitor_child = monitor.get('childrenIDs', []) + + if monitor_name == hostname: + for child in monitor_child: + child_monitor_name = api.get_monitor(child).get('name', 'Unknown') + print( + f"{child_monitor_name} does not exist anymore on the agent " + f"and has been deleted on {client_name} / {site_name} / {hostname}" + ) + safe_api_call(api.delete_monitor, child) + + time.sleep(1) + + NoSubMonitor = True + while NoSubMonitor: + relevant_monitors = api.get_monitors() + NoSubMonitor = False + + for monitor in relevant_monitors: + if monitor.get('type') == 'group': + monitor_name = monitor.get('name') + monitor_id = monitor.get('id') + children_ids = monitor.get('childrenIDs', []) + + if not children_ids: + print(f"{monitor_name} does not have any children") + safe_api_call(api.delete_monitor, monitor_id) + NoSubMonitor = True + + time.sleep(1) + + time.sleep(2) + + else: + print("Unexpected data format received.") + + else: + print(f"Request failed. Status code: {response.status_code}") + print(f"Error message: {response.text}") + + except requests.exceptions.RequestException as e: + print(f"An error occurred during the request: {e}") + + api.disconnect() diff --git a/scripts_staging/Checks/Gobetween status.sh b/scripts_staging/Checks/Gobetween status.sh new file mode 100644 index 00000000..7e941550 --- /dev/null +++ b/scripts_staging/Checks/Gobetween status.sh @@ -0,0 +1,107 @@ +#!/bin/bash +#SYNOPSIS +# Checks gobetween TCP load balancer health +# +#DESCRIPTION +# Monitors gobetween process, port listeners, backend connectivity, +# active connections, and recent errors. Runs via RMM agent. +# Returns Nagios-style exit codes (0=OK, 1=WARNING, 2=CRITICAL). +# +#PARAMETER debug +# Set env debug=1 for verbose per-backend test output to stderr +# +#NOTES +# Author: SAN +# Date: 02.06.26 +# #public +# +#CHANGELOG + + +STATUS=0 +OUTPUT="" +PERFDATA="" +CONFIG="/etc/gobetween/gobetween.toml" +PORTS=(80 443 25 993) +VERBOSE=false +[[ "${debug:-0}" == "1" || "${DEBUG:-0}" == "1" ]] && VERBOSE=true +# --- 1. Process check --- +PID=$(pgrep -x gobetween) +if [[ -z "$PID" ]]; then + echo "GOBETWEEN CRITICAL - gobetween process not running | active_conns=0" + exit 2 +fi +$VERBOSE && echo "[DEBUG] gobetween PID: $PID" >&2 +# --- 2. Port listeners --- +DOWN_PORTS=() +for PORT in "${PORTS[@]}"; do + ss -tlnp 2>/dev/null | grep -qE ":${PORT}\s" || DOWN_PORTS+=("$PORT") +done +$VERBOSE && echo "[DEBUG] expected ports: ${PORTS[*]}, down: ${DOWN_PORTS[*]:-none}" >&2 +if [[ ${#DOWN_PORTS[@]} -gt 0 ]]; then + OUTPUT+="ports down: ${DOWN_PORTS[*]}; " + STATUS=2 +fi +# --- 3. Backend connectivity --- +DOWN_BE=0 +BE_COUNT=0 +DOWN_LIST=() +while IFS= read -r line; do + trimmed=$(echo "$line" | sed 's/^[[:space:]]*//') + [[ -z "$trimmed" || "$trimmed" == \#* || "$trimmed" != *\"* ]] && continue + ip=$(echo "$line" | sed -n 's/.*"\([^"]*\)".*/\1/p') + [[ -z "$ip" || "$ip" != *":"* ]] && continue + BE_COUNT=$((BE_COUNT + 1)) + IP="${ip%:*}" + PORT="${ip##*:}" + $VERBOSE && echo -n "[DEBUG] testing $ip ... " >&2 + if ! timeout 3 bash -c "echo > /dev/tcp/$IP/$PORT" 2>/dev/null; then + DOWN_BE=$((DOWN_BE + 1)) + DOWN_LIST+=("$ip") + $VERBOSE && echo "FAIL" >&2 + else + $VERBOSE && echo "OK" >&2 + fi +done < <(grep -A50 'static_list' "$CONFIG" | grep '"') +$VERBOSE && echo "[DEBUG] $BE_COUNT backends tested, $DOWN_BE down" >&2 +if [[ $DOWN_BE -gt 0 ]]; then + PCT_DOWN=$((DOWN_BE * 100 / BE_COUNT)) + FAILS=$(IFS=,; echo "${DOWN_LIST[*]}") + if [[ $PCT_DOWN -ge 50 ]]; then + OUTPUT+="${DOWN_BE}/${BE_COUNT} backends unreachable [${FAILS}]; " + STATUS=2 + else + OUTPUT+="${DOWN_BE}/${BE_COUNT} backends unreachable [${FAILS}]; " + [[ $STATUS -eq 0 ]] && STATUS=1 + fi +fi +# --- 4. Active connections --- +ACTIVE_CONNS=$( (sudo netstat -tpn 2>/dev/null || ss -tpn 2>/dev/null) | grep -c "gobetween" ) +PERFDATA="active_conns=$ACTIVE_CONNS" +$VERBOSE && echo "[DEBUG] active connections: $ACTIVE_CONNS" >&2 +if [[ $ACTIVE_CONNS -gt 10000 ]]; then + OUTPUT+="connections high ($ACTIVE_CONNS); " + [[ $STATUS -eq 0 ]] && STATUS=1 +fi +# --- 5. Recent errors --- +ERROR_COUNT=$(journalctl -u gobetween --since "5 min ago" --no-pager 2>/dev/null | grep -cP '\[ERROR\]|\[PANIC\]|\[FATAL\]') +$VERBOSE && echo "[DEBUG] recent errors: $ERROR_COUNT" >&2 +if [[ $ERROR_COUNT -gt 10 ]]; then + OUTPUT+="$ERROR_COUNT errors in 5m; " + STATUS=2 +elif [[ $ERROR_COUNT -gt 0 ]]; then + OUTPUT+="$ERROR_COUNT errors in 5m; " + [[ $STATUS -eq 0 ]] && STATUS=1 +fi +# --- 6. Config file exists --- +if [[ ! -f "$CONFIG" ]]; then + OUTPUT+="config missing; " + STATUS=2 +fi +# --- Output --- +case $STATUS in + 0) echo "GOBETWEEN OK - all checks passed | $PERFDATA" ;; + 1) echo "GOBETWEEN WARNING - ${OUTPUT%; } | $PERFDATA" ;; + 2) echo "GOBETWEEN CRITICAL - ${OUTPUT%; } | $PERFDATA" ;; +esac +exit $STATUS \ No newline at end of file diff --git a/scripts_staging/TasksUpdater/Updater P3 Run SU.ps1 b/scripts_staging/TasksUpdater/Updater P3 Run SU.ps1 index bb255d86..798adf8e 100644 --- a/scripts_staging/TasksUpdater/Updater P3 Run SU.ps1 +++ b/scripts_staging/TasksUpdater/Updater P3 Run SU.ps1 @@ -39,12 +39,12 @@ 13.12.24 SAN Split logging from parser. 06.03.25 SAN added TRMM agent updater. 11.09.25 SAN disabled choco download progress output to shrink log size + 22.04.26 SAN added Chocolatey reboot detection from upgrade output .TODO Fix rename? #> - # Name will be used for both the name of the log file and what line of the Schedules to parse $PartName = "SoftwareUpdate" @@ -54,12 +54,10 @@ $PartName = "SoftwareUpdate" # Call the logging snippet env Company_folder_path will be passed {{Logging}} -# Function to check if a reboot is pending and return reasons function Get-PendingReboot { $rebootRequired = $false - $reasons = @() # Array to store reasons for reboot + $reasons = @() - # Check for Windows Update reboot required $WUReboot = Get-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\WindowsUpdate\Auto Update\RebootRequired" -ErrorAction SilentlyContinue if ($WUReboot) { $reasons += "Windows Update requires a reboot." @@ -67,21 +65,18 @@ function Get-PendingReboot { } # DISABLED DUE TO FALSE POSITIVE - # Check for pending file rename operations # $PendingFileRenameOperations = Get-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager" -Name "PendingFileRenameOperations" -ErrorAction SilentlyContinue if ($PendingFileRenameOperations) { $reasons += "Pending file rename operations require a reboot." $rebootRequired = $true } - # Check if Component-Based Servicing (CBS) requires a reboot $CBSReboot = Get-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Component Based Servicing\RebootPending" -ErrorAction SilentlyContinue if ($CBSReboot) { $reasons += "Component-Based Servicing requires a reboot." $rebootRequired = $true } - # Check for pending computer rename $ComputerRename = Get-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\ComputerName\ActiveComputerName" -ErrorAction SilentlyContinue $PendingComputerRename = Get-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\ComputerName\ComputerName" -ErrorAction SilentlyContinue if ($ComputerRename -and $PendingComputerRename -and ($ComputerRename.ComputerName -ne $PendingComputerRename.ComputerName)) { @@ -89,35 +84,31 @@ function Get-PendingReboot { $rebootRequired = $true } - # Check if Windows Installer (MSI) requires a reboot $PendingMSIReboot = Get-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\InProgress" -ErrorAction SilentlyContinue if ($PendingMSIReboot) { $reasons += "Windows Installer (MSI) operation requires a reboot." $rebootRequired = $true } - # Check if Group Policy client requires a reboot $GPReboot = Get-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Group Policy\State\Machine\RebootRequired" -ErrorAction SilentlyContinue if ($GPReboot) { $reasons += "Group Policy changes require a reboot." $rebootRequired = $true } - # Check for pending package installations $PendingPackageInstalls = Get-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Updates" -ErrorAction SilentlyContinue if ($PendingPackageInstalls) { $reasons += "Pending package installations require a reboot." $rebootRequired = $true } - # Return an object with reboot status and reasons return [PSCustomObject]@{ RebootRequired = $rebootRequired Reasons = $reasons } } -# check if reboot is needed +# check if reboot is needed BEFORE updates $result = Get-PendingReboot if ($result.RebootRequired) { Write-Host "Reboot is pending BEFORE updates for the following reasons:" @@ -126,11 +117,9 @@ if ($result.RebootRequired) { Write-Host "No Reboot is pending BEFORE updates." } -# The following section is in place due to the fact that ps logging does not capture RAW output from choco please do not touch -# List outdated packages and capture output +# Chocolatey output capture $outdatedPackages = choco outdated | Out-String -# Upgrade all packages and capture output -$upgradeResult = choco upgrade all -y --no-progress| Out-String +$upgradeResult = choco upgrade all -y --no-progress | Out-String Write-Host "" Write-Host "------------------------------------------------------------" @@ -143,13 +132,20 @@ Write-Host $upgradeResult Write-Host "" Write-Host "------------------------------------------------------------" Write-Host "" + Write-Host "------------------------------------------------------------" Write-Host "TRMM Agent update" {{Update TRMM agent}} - -# Check if a reboot is pending and reboot if necessary +# Check again for reboot AFTER update $result = Get-PendingReboot + +if ($upgradeResult -match "A pending system reboot request has been detected") { + Write-Host "Reboot condition detected in Chocolatey output." + $result.RebootRequired = $true + $result.Reasons += "Chocolatey reports a pending reboot request." +} + if ($result.RebootRequired) { Write-Host "Reboot is pending AFTER update for the following reasons:" $result.Reasons | ForEach-Object { Write-Host "- $_" } @@ -158,12 +154,10 @@ if ($result.RebootRequired) { $timeDifference = New-TimeSpan -Start (Get-Date) -End $scheduledTime $SetReboot = [int]$timeDifference.TotalSeconds - # Schedule the system reboot Write-Host "shutdown.exe /r /f /t $SetReboot /c Reboot done by RMM task, required after packages updates /d p:4:1" shutdown.exe /r /f /t $SetReboot /c "Reboot done by RMM task, required after packages updates" /d p:4:1 - - # Output a warning message - $minutes = [math]::Floor($SetReboot / 60) # Rounding + + $minutes = [math]::Floor($SetReboot / 60) $Message = "The system will reboot in $minutes minutes. Please save your work." Write-Host $Message msg * $Message