Skip to content

make main.go code more fault tolerant #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nadirhamid merged 1 commit into from
Jul 16, 2026
Merged

make main.go code more fault tolerant #14

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 105 additions & 86 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,44 @@
package main

import (
"context"
"fmt"
"net/http"
"os"
"os/signal"
"sync"
"syscall"
"time"
//"errors"
"github.com/gocql/gocql"
"fmt"

helpers "github.com/Lineblocs/go-helpers"
_ "github.com/go-sql-driver/mysql"
"github.com/go-redis/redis"
"github.com/gocql/gocql"
"github.com/labstack/echo/v4"
"github.com/mrwaggel/golimiter"
"github.com/go-redis/redis"
"github.com/sirupsen/logrus"
"lineblocs.com/api/database"
"lineblocs.com/api/handler"
"lineblocs.com/api/model"
"lineblocs.com/api/router"
"lineblocs.com/api/store"
"lineblocs.com/api/utils"
"lineblocs.com/api/database"
)

// Global service variables
var dbConn *database.MySQLConn
var rdb *redis.Client
var cqlCluster *gocql.ClusterConfig
var cqlSess *gocql.Session
var data *model.ServerData
var customizations *helpers.CustomizationSettings

func updateCustomizationSettings() (error) {
// updateCustomizationSettings fetches configuration updates from helper utilities.
// Any retrieval failure is logged and returned as an error to be handled by the caller.
func updateCustomizationSettings() error {
record, err := helpers.GetCustomizationSettings()
if err != nil {
utils.Log(logrus.PanicLevel, err.Error())
panic(err)
utils.Log(logrus.ErrorLevel, err.Error())
return err
}

Expand All @@ -42,103 +47,138 @@ func updateCustomizationSettings() (error) {
}

func main() {
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()

// Init Logrus and configure channels
// Initialize Logrus and configure targets based on configurations
logDestination := utils.Config("LOG_DESTINATIONS")
helpers.InitLogrus(logDestination)

utils.Log(logrus.InfoLevel, "Running setup methods for api server..")
// Load media_server list from db and create media server
var err error

// Fetch and load media_server listing. Clean exit on error to allow orchestrator (K8s/Docker) to restart.
servers, err := helpers.CreateMediaServers()
if err != nil {
utils.Log(logrus.ErrorLevel, fmt.Sprintf("Failed to create media servers: %v", err))
os.Exit(1)
}

data = &model.ServerData{
Mutex: sync.RWMutex{},
Servers: servers}

if err != nil {
utils.Log(logrus.PanicLevel, err.Error())
panic(err)
Servers: servers,
}

// Create DB Connection with MySQL
// Establish connection with the MySQL database.
utils.Log(logrus.InfoLevel, "Connecting to database...")
db, err := helpers.CreateDBConn()
if err != nil {
utils.Log(logrus.PanicLevel, err.Error())
panic(err)
utils.Log(logrus.ErrorLevel, fmt.Sprintf("Failed to connect to database: %v", err))
os.Exit(1)
}

dbConn = database.NewMySQLConn(db)

// Establish connection with the Redis cache.
rdb, err = helpers.CreateRedisConn()

// get customization record before starting server
err = updateCustomizationSettings()
if err != nil {
utils.Log(logrus.PanicLevel, err.Error())
panic(err)
utils.Log(logrus.ErrorLevel, fmt.Sprintf("Failed to connect to Redis: %v", err))
os.Exit(1)
}

// connect to cassandra
/*
utils.Log(logrus.InfoLevel, "Connecting to cassandra...")
cassandraAddr := utils.Config("CASSANDRA_HOST") + ":9042"
cqlCluster = gocql.NewCluster(cassandraAddr)
cqlCluster.Keyspace = utils.Config("CASSANDRA_KEYSPACE")
cqlCluster.ProtoVersion = 4
cqlSess, err = cqlCluster.CreateSession()
// Retrieve primary configuration records prior to triggering routing.
err = updateCustomizationSettings()
if err != nil {
utils.Log(logrus.PanicLevel, err.Error())
panic(err)
utils.Log(logrus.ErrorLevel, fmt.Sprintf("Failed to fetch initial customizations: %v", err))
os.Exit(1)
}
*/

utils.Log(logrus.InfoLevel, fmt.Sprintf("got customization settings. billing frequency = %s", customizations.BillingFrequency))

var wg sync.WaitGroup
wg.Add(1)
// Setup context for background routines to stop cleanly during shutdown.
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

// Initialize background sync intervals
ticker := time.NewTicker(60 * time.Second)
defer ticker.Stop()

// Spin off asynchronous settings-update routine
go func() {
for {
utils.Log(logrus.InfoLevel, fmt.Sprintf("updating customization settings"))
<-ticker.C
err := updateCustomizationSettings()
if err != nil {
utils.Log(logrus.DebugLevel, fmt.Sprintf("failed to get updated customizations record"))
continue
for {
select {
case <-ticker.C:
utils.Log(logrus.InfoLevel, "updating customization settings")
err := updateCustomizationSettings()
if err != nil {
utils.Log(logrus.DebugLevel, "failed to get updated customizations record")
continue
}
utils.Log(logrus.InfoLevel, "updated settings successfully.")
case <-ctx.Done():
utils.Log(logrus.InfoLevel, "stopping customization settings updater routine")
return
}
}
}()

utils.Log(logrus.InfoLevel, fmt.Sprintf("updated settings successfully."))
}
}()
// Construct HTTP router framework and inject data stores
r := buildServer()

// Spin up Echo engine in a non-blocking goroutine
go func() {
// Start Internals-API Backend server
utils.Log(logrus.InfoLevel, "Starting API...")
startServer()
wg.Done()
if utils.Config("USE_TLS") == "on" {
certPath := utils.Config("TLS_CERT_PATH")
keyPath := utils.Config("TLS_KEY_PATH")
httpsPort := utils.ReadEnv("HTTPS_PORT", "443")
utils.Log(logrus.InfoLevel, fmt.Sprintf("Starting HTTP server with TLS. cert=%s, key=%s\r\n", certPath, keyPath))
if err := r.StartTLS(":"+httpsPort, certPath, keyPath); err != nil && err != http.ErrServerClosed {
utils.Log(logrus.ErrorLevel, fmt.Sprintf("TLS server error: %v", err))
os.Exit(1)
}
} else {
httpPort := utils.ReadEnv("HTTP_PORT", "80")
utils.Log(logrus.InfoLevel, fmt.Sprintf("HTTP port %s\r\n", httpPort))
if err := r.Start(":" + httpPort); err != nil && err != http.ErrServerClosed {
utils.Log(logrus.ErrorLevel, fmt.Sprintf("HTTP server error: %v", err))
os.Exit(1)
}
}
}()
wg.Wait()

// Monitor OS termination signals to coordinate shutdown
shutdownSig := make(chan os.Signal, 1)
signal.Notify(shutdownSig, syscall.SIGINT, syscall.SIGTERM)

// Wait for termination signal
<-shutdownSig
utils.Log(logrus.InfoLevel, "Shutting down service gracefully...")

// Cancel background routines immediately
cancel()

// Implement graceful shutdown with safety timeout context (10 seconds)
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer shutdownCancel()

if err := r.Shutdown(shutdownCtx); err != nil {
utils.Log(logrus.ErrorLevel, fmt.Sprintf("Server forced to shutdown: %v", err))
os.Exit(1)
}

utils.Log(logrus.InfoLevel, "Service stopped cleanly.")
os.Exit(0)
}

// Start Internals-API Backend server
// Configure Handler, limit middleware, TLS
func startServer() {
// buildServer constructs the Echo framework, creates stores, and configures route targets
func buildServer() *echo.Echo {
utils.SetSetting(model.GlobalSettings{ValidateCallerId: false})

// Start Server with Echo
r := router.New()
utils.Log(logrus.InfoLevel, "Starting HTTP server...")
// Configure Limit Handler if USE_LIMIT_MIDDLEWARE is "on"
utils.Log(logrus.InfoLevel, "Configuring HTTP server...")

if utils.Config("USE_LIMIT_MIDDLEWARE") == "on" {
r.Any("", limitHandler)
}

// Configure Handler with Global DB
// Instantiate operational store nodes
as := store.NewAdminStore(dbConn)
cs := store.NewCallStore(dbConn)
crs := store.NewCarrierStore(dbConn)
Expand All @@ -149,29 +189,12 @@ func startServer() {
us := store.NewUserStore(dbConn, rdb)
h := handler.NewHandler(as, cs, crs, ds, fs, ls, rs, us)

// Register Handler for Echo context
// Bind endpoints
h.Register(r)

// Start with 443 port if TLS is ON
utils.Log(logrus.InfoLevel, "Starting HTTP server without TLS\r\n")
if utils.Config("USE_TLS") == "on" {
certPath := utils.Config("TLS_CERT_PATH")
keyPath := utils.Config("TLS_KEY_PATH")
httpsPort := utils.ReadEnv("HTTPS_PORT", "443")
utils.Log(logrus.InfoLevel, fmt.Sprintf("Starting HTTP server with TLS. cert=%s, key=%s\r\n", certPath, keyPath))
r.Logger.Fatal(r.StartTLS(":"+httpsPort, certPath, keyPath))
utils.Log(logrus.InfoLevel, "Started server...")
return
}

// Start with 80 port if TLS is OFF
httpPort := utils.ReadEnv("HTTP_PORT", "80")
utils.Log(logrus.InfoLevel, fmt.Sprintf("HTTP port %s\r\n", httpPort))
r.Logger.Fatal(r.Start(":" + httpPort))
utils.Log(logrus.InfoLevel, "Started server...")
return r
}

// Configure Limit Handler for Echo context
// limitHandler intercepts API calls to prevent brute force or denial attempts
func limitHandler(c echo.Context) error {
var addr string
requestedAddr := c.QueryParam("addr")
Expand All @@ -188,20 +211,16 @@ func limitHandler(c echo.Context) error {
isCarrier = utils.CheckIfCarrier(carrier)
}

// Limit for users

var limit int = 60
if isCarrier {
limit = 3600
}

var indexLimiter = golimiter.New(limit, time.Minute)

// Check if the given IP is rate limited
if indexLimiter.IsLimited(addr) {
return c.String(http.StatusTooManyRequests, fmt.Sprintf("Rate limit exhausted from %s", addr))
}
// Add a request to the count for the Ip
indexLimiter.Increment(addr)
totalRequestPastMinute := indexLimiter.Count(addr)
totalRemaining := limit - totalRequestPastMinute
Expand All @@ -211,4 +230,4 @@ func limitHandler(c echo.Context) error {
"You are allowed to make %d more request.\n"+
"Maximum request you can make per minute is %d.",
addr, totalRequestPastMinute, totalRemaining, limit))
}
}
Loading