From 4274a71152e44a2c5c2a23708341ae89f40c2f86 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 19:52:45 -0500 Subject: [PATCH 01/49] removing database dependencies --- Dockerfile | 11 +- database/concurrency_utils.go | 56 ----- database/database_test.go | 100 --------- database/db.go | 308 --------------------------- go.mod | 9 +- go.sum | 14 +- handlers/m3u_handler.go | 18 ++ handlers/stream_handler.go | 102 +++++++++ m3u/downloader.go | 57 ----- m3u/generate.go | 245 ---------------------- m3u/m3u_test.go | 191 ----------------- m3u/middlewares.go | 47 ----- m3u/parser.go | 259 ----------------------- main.go | 23 +- proxy/load_balancer.go | 98 +++++++++ proxy/proxy_stream.go | 184 ++++++++++++++++ proxy/stream_handler.go | 370 --------------------------------- store/cache.go | 166 +++++++++++++++ store/concurrency.go | 72 +++++++ store/downloader.go | 90 ++++++++ {m3u => store}/filters.go | 5 +- store/parser.go | 122 +++++++++++ store/slug.go | 58 ++++++ store/streams.go | 116 +++++++++++ {database => store}/types.go | 2 +- {proxy => tests}/proxy_test.go | 42 ++-- updater/updater.go | 82 ++------ utils/checksum.go | 11 + utils/m3u_path.go | 19 ++ utils/middlewares.go | 46 ++++ utils/stream_url.go | 31 +++ 31 files changed, 1179 insertions(+), 1775 deletions(-) delete mode 100644 database/concurrency_utils.go delete mode 100644 database/database_test.go delete mode 100644 database/db.go create mode 100644 handlers/m3u_handler.go create mode 100644 handlers/stream_handler.go delete mode 100644 m3u/downloader.go delete mode 100644 m3u/generate.go delete mode 100644 m3u/m3u_test.go delete mode 100644 m3u/middlewares.go delete mode 100644 m3u/parser.go create mode 100644 proxy/load_balancer.go create mode 100644 proxy/proxy_stream.go delete mode 100644 proxy/stream_handler.go create mode 100644 store/cache.go create mode 100644 store/concurrency.go create mode 100644 store/downloader.go rename {m3u => store}/filters.go (92%) create mode 100644 store/parser.go create mode 100644 store/slug.go create mode 100644 store/streams.go rename {database => store}/types.go (91%) rename {proxy => tests}/proxy_test.go (62%) create mode 100644 utils/checksum.go create mode 100644 utils/m3u_path.go create mode 100644 utils/middlewares.go create mode 100644 utils/stream_url.go diff --git a/Dockerfile b/Dockerfile index a2b77958..be8e1b3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,16 +13,9 @@ RUN go mod download # Copy the source code from the current directory to the Working Directory inside the container COPY . . -# fire up redis server and test and build the app. +# test and build the app. # hadolint ignore=DL3018 -RUN \ - if [ "$(uname -m)" = "x86_64" ]; then \ - apk --no-cache add redis && \ - sed -i "s/bind .*/bind 127.0.0.1/g" /etc/redis.conf && \ - redis-server --daemonize yes && \ - go test ./...; \ - fi && \ - go build -ldflags='-s -w' -o m3u-proxy . +RUN go test ./tests/... && go build -ldflags='-s -w' -o m3u-proxy . # End from the latest alpine image # hadolint ignore=DL3007 diff --git a/database/concurrency_utils.go b/database/concurrency_utils.go deleted file mode 100644 index 61c9ea19..00000000 --- a/database/concurrency_utils.go +++ /dev/null @@ -1,56 +0,0 @@ -package database - -import ( - "fmt" - "m3u-stream-merger/utils" - "os" - "strconv" -) - -func (db *Instance) ConcurrencyPriorityValue(m3uIndex int) int { - maxConcurrency, err := strconv.Atoi(os.Getenv(fmt.Sprintf("M3U_MAX_CONCURRENCY_%d", m3uIndex+1))) - if err != nil { - maxConcurrency = 1 - } - - count, err := db.GetConcurrency(m3uIndex) - if err != nil { - count = 0 - } - - return maxConcurrency - count -} - -func (db *Instance) CheckConcurrency(m3uIndex int) bool { - maxConcurrency, err := strconv.Atoi(os.Getenv(fmt.Sprintf("M3U_MAX_CONCURRENCY_%d", m3uIndex+1))) - if err != nil { - maxConcurrency = 1 - } - - count, err := db.GetConcurrency(m3uIndex) - if err != nil { - utils.SafeLogf("Error checking concurrency: %s\n", err.Error()) - return false - } - - utils.SafeLogf("Current number of connections for M3U_%d: %d", m3uIndex+1, count) - return count >= maxConcurrency -} - -func (db *Instance) UpdateConcurrency(m3uIndex int, incr bool) { - var err error - if incr { - err = db.IncrementConcurrency(m3uIndex) - } else { - err = db.DecrementConcurrency(m3uIndex) - } - if err != nil { - utils.SafeLogf("Error updating concurrency: %s\n", err.Error()) - } - - count, err := db.GetConcurrency(m3uIndex) - if err != nil { - utils.SafeLogf("Error checking concurrency: %s\n", err.Error()) - } - utils.SafeLogf("Current number of connections for M3U_%d: %d", m3uIndex+1, count) -} diff --git a/database/database_test.go b/database/database_test.go deleted file mode 100644 index 687e0b2f..00000000 --- a/database/database_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package database - -import ( - "os" - "testing" -) - -func TestSaveAndLoadFromDb(t *testing.T) { - // Test InitializeDb and check if the database file exists - os.Setenv("REDIS_DB", "1") - - db, err := InitializeDb() - if err != nil { - t.Errorf("InitializeDb returned error: %v", err) - } - - err = db.ClearDb() - if err != nil { - t.Errorf("ClearDb returned error: %v", err) - } - - // Test LoadFromDb with existing data in the database - expected := []*StreamInfo{{ - Slug: "stream1", - Title: "stream1", - TvgID: "test1", - LogoURL: "http://test.com/image.png", - Group: "test", - URLs: map[int]string{0: "testing"}, - }, { - Slug: "stream2", - Title: "stream2", - TvgID: "test2", - LogoURL: "http://test2.com/image.png", - Group: "test2", - URLs: map[int]string{0: "testing2"}, - }} - - err = db.SaveToDb(expected) // Insert test data into the database - if err != nil { - t.Errorf("SaveToDb returned error: %v", err) - } - - streamChan := db.GetStreams() - - var result []StreamInfo - for stream := range streamChan { - result = append(result, stream) - } - - if len(result) != len(expected) { - t.Errorf("GetStreams returned %+v, expected %+v", result, expected) - } - - for i, expectedStream := range expected { - if !streamInfoEqual(result[i], *expectedStream) { - t.Errorf("GetStreams returned %+v, expected %+v", result[i], expectedStream) - } - } - - err = db.DeleteStreamBySlug(expected[1].Slug) - if err != nil { - t.Errorf("DeleteStreamBySlug returned error: %v", err) - } - - streamChan = db.GetStreams() - - result = []StreamInfo{} - for stream := range streamChan { - result = append(result, stream) - } - - expected = expected[:1] - - if len(result) != len(expected) { - t.Errorf("GetStreams returned %+v, expected %+v", result, expected) - } - - for i, expectedStream := range expected { - if !streamInfoEqual(result[i], *expectedStream) { - t.Errorf("GetStreams returned %+v, expected %+v", result[i], expectedStream) - } - } - -} - -// streamInfoEqual checks if two StreamInfo objects are equal. -func streamInfoEqual(a, b StreamInfo) bool { - if a.Slug != b.Slug || a.TvgID != b.TvgID || a.Title != b.Title || a.Group != b.Group || a.LogoURL != b.LogoURL || len(a.URLs) != len(b.URLs) { - return false - } - - for i, url := range a.URLs { - if url != b.URLs[i] { - return false - } - } - - return true -} diff --git a/database/db.go b/database/db.go deleted file mode 100644 index b1b54671..00000000 --- a/database/db.go +++ /dev/null @@ -1,308 +0,0 @@ -package database - -import ( - "context" - "encoding/json" - "fmt" - "m3u-stream-merger/utils" - "math" - "os" - "strconv" - "strings" - "time" - - "github.com/redis/go-redis/v9" -) - -type Instance struct { - Redis *redis.Client - Ctx context.Context -} - -func InitializeDb() (*Instance, error) { - ctx := context.Background() - - addr := os.Getenv("REDIS_ADDR") - password := os.Getenv("REDIS_PASS") - db := 0 - if i, err := strconv.Atoi(os.Getenv("REDIS_DB")); err == nil { - db = i - } - - var redisOptions *redis.Options - - if password == "" { - redisOptions = &redis.Options{ - Addr: addr, - DB: db, - DialTimeout: 10 * time.Second, - ReadTimeout: 1 * time.Minute, - WriteTimeout: 1 * time.Minute, - } - } else { - redisOptions = &redis.Options{ - Addr: addr, - Password: password, - DB: db, - DialTimeout: 10 * time.Second, - ReadTimeout: 1 * time.Minute, - WriteTimeout: 1 * time.Minute, - } - } - - redisInstance := redis.NewClient(redisOptions) - - for { - err := redisInstance.Ping(ctx).Err() - if err == nil { - break - } - fmt.Printf("Error connecting to Redis: %v. Retrying...\n", err) - time.Sleep(2 * time.Second) - } - - return &Instance{Redis: redisInstance, Ctx: ctx}, nil -} - -func (db *Instance) ClearDb() error { - if err := db.Redis.FlushDB(db.Ctx).Err(); err != nil { - return fmt.Errorf("error clearing Redis: %v", err) - } - - return nil -} - -func (db *Instance) SaveToDb(streams []*StreamInfo) error { - var debug = os.Getenv("DEBUG") == "true" - - pipeline := db.Redis.Pipeline() - - pipeline.FlushDB(db.Ctx) - - for _, s := range streams { - streamKey := fmt.Sprintf("stream:%s", s.Slug) - - streamDataJson, err := json.Marshal(s) - if err != nil { - return fmt.Errorf("SaveToDb error: %v", err) - } - - if debug { - utils.SafeLogf("[DEBUG] Preparing to set data for stream key %s: %v\n", streamKey, s) - } - - pipeline.Set(db.Ctx, streamKey, string(streamDataJson), 0) - - // Add to the sorted set - sortScore := calculateSortScore(*s) - - if debug { - utils.SafeLogf("[DEBUG] Adding to sorted set with score %f and member %s\n", sortScore, streamKey) - } - - pipeline.ZAdd(db.Ctx, "streams_sorted", redis.Z{ - Score: sortScore, - Member: streamKey, - }) - } - - if len(streams) > 0 { - if debug { - utils.SafeLogln("[DEBUG] Executing pipeline...") - } - - _, err := pipeline.Exec(db.Ctx) - if err != nil { - return fmt.Errorf("SaveToDb error: %v", err) - } - - if debug { - utils.SafeLogln("[DEBUG] Pipeline executed successfully.") - } - } - - return nil -} -func (db *Instance) DeleteStreamBySlug(slug string) error { - streamKey := fmt.Sprintf("stream:%s", slug) - - // Delete from the sorted set - if err := db.Redis.ZRem(db.Ctx, "streams_sorted", streamKey).Err(); err != nil { - return fmt.Errorf("error removing stream from sorted set: %v", err) - } - - // Delete the stream itself - if err := db.Redis.Del(db.Ctx, streamKey).Err(); err != nil { - return fmt.Errorf("error deleting stream from Redis: %v", err) - } - - return nil -} - -func (db *Instance) GetStreamBySlug(slug string) (StreamInfo, error) { - streamKey := fmt.Sprintf("stream:%s", slug) - streamDataJson, err := db.Redis.Get(db.Ctx, streamKey).Result() - if err != nil { - return StreamInfo{}, fmt.Errorf("error getting stream from Redis: %v", err) - } - - stream := StreamInfo{} - - err = json.Unmarshal([]byte(streamDataJson), &stream) - if err != nil { - return StreamInfo{}, fmt.Errorf("error getting stream: %v", err) - } - - if strings.TrimSpace(stream.Title) == "" { - return StreamInfo{}, fmt.Errorf("stream not found: %s", slug) - } - - return stream, nil -} - -func (db *Instance) GetStreams() <-chan StreamInfo { - var debug = os.Getenv("DEBUG") == "true" - - // Channels for streaming results and errors - streamChan := make(chan StreamInfo) - - go func() { - defer close(streamChan) - - keys, err := db.Redis.ZRange(db.Ctx, "streams_sorted", 0, -1).Result() - if err != nil { - utils.SafeLogf("error retrieving streams: %v", err) - return - } - - // Filter out URL keys - for _, key := range keys { - streamDataJson, err := db.Redis.Get(db.Ctx, key).Result() - if err != nil { - utils.SafeLogf("error retrieving stream data: %v", err) - return - } - - stream := StreamInfo{} - - err = json.Unmarshal([]byte(streamDataJson), &stream) - if err != nil { - utils.SafeLogf("error retrieving stream data: %v", err) - return - } - - if debug { - utils.SafeLogf("[DEBUG] Processing stream: %v\n", stream) - } - - // Send the stream to the channel - streamChan <- stream - } - - if debug { - utils.SafeLogln("[DEBUG] Streams retrieved successfully.") - } - }() - - return streamChan -} - -// GetConcurrency retrieves the concurrency count for the given m3uIndex -func (db *Instance) GetConcurrency(m3uIndex int) (int, error) { - key := "concurrency:" + strconv.Itoa(m3uIndex) - countStr, err := db.Redis.Get(db.Ctx, key).Result() - if err == redis.Nil { - return 0, nil // Key does not exist - } else if err != nil { - return 0, err - } - - count, err := strconv.Atoi(countStr) - if err != nil { - return 0, err - } - - return count, nil -} - -// IncrementConcurrency increments the concurrency count for the given m3uIndex -func (db *Instance) IncrementConcurrency(m3uIndex int) error { - key := "concurrency:" + strconv.Itoa(m3uIndex) - return db.Redis.Incr(db.Ctx, key).Err() -} - -// DecrementConcurrency decrements the concurrency count for the given m3uIndex -func (db *Instance) DecrementConcurrency(m3uIndex int) error { - key := "concurrency:" + strconv.Itoa(m3uIndex) - return db.Redis.Decr(db.Ctx, key).Err() -} - -func (db *Instance) ClearConcurrencies() error { - var cursor uint64 - var err error - var keys []string - - for { - var scanKeys []string - scanKeys, cursor, err = db.Redis.Scan(db.Ctx, cursor, "concurrency:*", 0).Result() - if err != nil { - return fmt.Errorf("error scanning keys from Redis: %v", err) - } - keys = append(keys, scanKeys...) - if cursor == 0 { - break - } - } - - if len(keys) == 0 { - return nil - } - - if err := db.Redis.Del(db.Ctx, keys...).Err(); err != nil { - return fmt.Errorf("error deleting keys from Redis: %v", err) - } - - return nil -} - -func getSortingValue(s StreamInfo) string { - key := os.Getenv("SORTING_KEY") - - var value string - switch key { - case "tvg-id": - value = s.TvgID - case "tvg-chno": - value = s.TvgChNo - default: - value = s.TvgID - } - - // Try to parse the value as a float. - if numValue, err := strconv.ParseFloat(value, 64); err == nil { - return fmt.Sprintf("%010.2f", numValue) + s.Title - } - - // If parsing fails, fall back to using the original string value. - return value + s.Title -} - -func calculateSortScore(s StreamInfo) float64 { - // Add to the sorted set with tvg_id as the score - maxLen := 40 - base := float64(256) - - // Normalize length by padding the string - paddedString := strings.ToLower(getSortingValue(s)) - if len(paddedString) < maxLen { - paddedString = paddedString + strings.Repeat("\x00", maxLen-len(paddedString)) - } - - sortScore := 0.0 - for i := 0; i < len(paddedString); i++ { - charValue := float64(paddedString[i]) - sortScore += charValue / math.Pow(base, float64(i+1)) - } - - return sortScore -} diff --git a/go.mod b/go.mod index 05702c94..d9771822 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,10 @@ module m3u-stream-merger go 1.23.0 require ( + github.com/goccy/go-json v0.10.3 github.com/gosimple/slug v1.14.0 - github.com/redis/go-redis/v9 v9.7.0 + github.com/klauspost/compress v1.17.11 github.com/robfig/cron/v3 v3.0.1 ) -require ( - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/gosimple/unidecode v1.0.1 // indirect -) +require github.com/gosimple/unidecode v1.0.1 // indirect diff --git a/go.sum b/go.sum index 4cfc4ca2..13983400 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,10 @@ -github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= -github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= -github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gosimple/slug v1.14.0 h1:RtTL/71mJNDfpUbCOmnf/XFkzKRtD6wL6Uy+3akm4Es= github.com/gosimple/slug v1.14.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= -github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go new file mode 100644 index 00000000..8ebbe91b --- /dev/null +++ b/handlers/m3u_handler.go @@ -0,0 +1,18 @@ +package handlers + +import ( + "m3u-stream-merger/store" + "m3u-stream-merger/utils" + "net/http" +) + +func M3UHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.Header().Set("Access-Control-Allow-Origin", "*") + + content := store.RevalidatingGetM3U(r) + + if _, err := w.Write([]byte(content)); err != nil { + utils.SafeLogf("[ERROR] Failed to write response: %v\n", err) + } +} diff --git a/handlers/stream_handler.go b/handlers/stream_handler.go new file mode 100644 index 00000000..2c4588d6 --- /dev/null +++ b/handlers/stream_handler.go @@ -0,0 +1,102 @@ +package handlers + +import ( + "context" + "m3u-stream-merger/proxy" + "m3u-stream-merger/store" + "m3u-stream-merger/utils" + "net/http" + "os" + "path" + "strings" +) + +func StreamHandler(w http.ResponseWriter, r *http.Request, cm *store.ConcurrencyManager) { + debug := os.Getenv("DEBUG") == "true" + + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + utils.SafeLogf("Received request from %s for URL: %s\n", r.RemoteAddr, r.URL.Path) + + streamUrl := strings.Split(path.Base(r.URL.Path), ".")[0] + if streamUrl == "" { + utils.SafeLogf("Invalid m3uID for request from %s: %s\n", r.RemoteAddr, r.URL.Path) + http.NotFound(w, r) + return + } + + stream, err := proxy.NewStreamInstance(strings.TrimPrefix(streamUrl, "/"), cm) + if err != nil { + utils.SafeLogf("Error retrieving stream for slug %s: %v\n", streamUrl, err) + http.NotFound(w, r) + return + } + + var selectedIndex int + var selectedUrl string + + testedIndexes := []int{} + firstWrite := true + + var resp *http.Response + + for { + select { + case <-ctx.Done(): + utils.SafeLogf("Client disconnected: %s\n", r.RemoteAddr) + return + default: + resp, selectedUrl, selectedIndex, err = stream.LoadBalancer(&testedIndexes, r.Method) + if err != nil { + utils.SafeLogf("Error reloading stream for %s: %v\n", streamUrl, err) + return + } + + // HTTP header initialization + if firstWrite { + for k, v := range resp.Header { + if strings.ToLower(k) == "content-length" { + continue + } + + for _, val := range v { + w.Header().Set(k, val) + } + } + w.WriteHeader(resp.StatusCode) + + if debug { + utils.SafeLogf("[DEBUG] Headers set for response: %v\n", w.Header()) + } + firstWrite = false + } + + exitStatus := make(chan int) + + utils.SafeLogf("Proxying %s to %s\n", r.RemoteAddr, selectedUrl) + go stream.ProxyStream(ctx, selectedIndex, resp, r, w, exitStatus) + testedIndexes = append(testedIndexes, selectedIndex) + + streamExitCode := <-exitStatus + utils.SafeLogf("Exit code %d received from %s\n", streamExitCode, selectedUrl) + + if streamExitCode == 2 && utils.EOFIsExpected(resp) { + utils.SafeLogf("Successfully proxied playlist: %s\n", r.RemoteAddr) + cancel() + } else if streamExitCode == 1 || streamExitCode == 2 { + // Retry on server-side connection errors + utils.SafeLogf("Retrying other servers...\n") + } else if streamExitCode == 4 { + utils.SafeLogf("Finished handling %s request: %s\n", r.Method, r.RemoteAddr) + cancel() + } else { + // Consider client-side connection errors as complete closure + utils.SafeLogf("Client has closed the stream: %s\n", r.RemoteAddr) + cancel() + } + + resp.Body.Close() + } + } +} diff --git a/m3u/downloader.go b/m3u/downloader.go deleted file mode 100644 index 35a68115..00000000 --- a/m3u/downloader.go +++ /dev/null @@ -1,57 +0,0 @@ -package m3u - -import ( - "bytes" - "fmt" - "io" - "os" - "strings" - - "m3u-stream-merger/utils" -) - -func downloadM3UToBuffer(m3uURL string, buffer *bytes.Buffer) (err error) { - debug := os.Getenv("DEBUG") == "true" - if debug { - utils.SafeLogf("[DEBUG] Downloading M3U from: %s\n", m3uURL) - } - - var file io.Reader - - if strings.HasPrefix(m3uURL, "file://") { - localPath := strings.TrimPrefix(m3uURL, "file://") - utils.SafeLogf("Reading M3U from local file: %s\n", localPath) - - localFile, err := os.Open(localPath) - if err != nil { - return fmt.Errorf("Error opening file: %v", err) - } - defer localFile.Close() - - file = localFile - } else { - utils.SafeLogf("Downloading M3U from URL: %s\n", m3uURL) - resp, err := utils.CustomHttpRequest("GET", m3uURL) - if err != nil { - return fmt.Errorf("HTTP GET error: %v", err) - } - - defer func() { - _, _ = io.Copy(io.Discard, resp.Body) - resp.Body.Close() - }() - - file = resp.Body - } - - _, err = io.Copy(buffer, file) - if err != nil { - return fmt.Errorf("Error reading file: %v", err) - } - - if debug { - utils.SafeLogln("[DEBUG] Successfully copied M3U content to buffer") - } - - return nil -} diff --git a/m3u/generate.go b/m3u/generate.go deleted file mode 100644 index 08d63cf0..00000000 --- a/m3u/generate.go +++ /dev/null @@ -1,245 +0,0 @@ -package m3u - -import ( - "fmt" - "m3u-stream-merger/database" - "m3u-stream-merger/utils" - "net/http" - "net/url" - "os" - "path" - "strings" - "sync" -) - -type Cache struct { - sync.Mutex - data string - Revalidating bool -} - -var M3uCache = &Cache{} - -const cacheFilePath = "/m3u-proxy/cache.m3u" - -func InitCache(db *database.Instance) { - debug := isDebugMode() - - M3uCache.Lock() - if M3uCache.Revalidating { - M3uCache.Unlock() - if debug { - utils.SafeLogln("[DEBUG] Cache revalidation is already in progress. Skipping.") - } - return - } - M3uCache.Revalidating = true - M3uCache.Unlock() - - content := GenerateAndCacheM3UContent(db, nil) - err := WriteCacheToFile(content) - if err != nil { - utils.SafeLogf("Error writing cache to file: %v\n", err) - } -} - -func isDebugMode() bool { - return os.Getenv("DEBUG") == "true" -} - -func getFileExtensionFromUrl(rawUrl string) (string, error) { - u, err := url.Parse(rawUrl) - if err != nil { - return "", err - } - return path.Ext(u.Path), nil -} - -func getSubPathFromUrl(rawUrl string) (string, error) { - parsedURL, err := url.Parse(rawUrl) - if err != nil { - return "", err - } - - pathSegments := strings.Split(parsedURL.Path, "/") - - if len(pathSegments) <= 1 { - return "stream", nil - } - - basePath := strings.Join(pathSegments[1:len(pathSegments)-1], "/") - return basePath, nil -} - -func GenerateStreamURL(baseUrl string, stream database.StreamInfo) string { - var subPath string - var err error - for _, srcUrl := range stream.URLs { - subPath, err = getSubPathFromUrl(srcUrl) - if err != nil { - continue - } - - ext, err := getFileExtensionFromUrl(srcUrl) - if err != nil { - return fmt.Sprintf("%s/proxy/%s/%s\n", baseUrl, subPath, stream.Slug) - } - - return fmt.Sprintf("%s/proxy/%s/%s%s\n", baseUrl, subPath, stream.Slug, ext) - } - return fmt.Sprintf("%s/proxy/stream/%s\n", baseUrl, stream.Slug) -} - -func GenerateAndCacheM3UContent(db *database.Instance, r *http.Request) string { - debug := isDebugMode() - if debug { - utils.SafeLogln("[DEBUG] Regenerating M3U cache in the background") - } - - baseUrl := utils.DetermineBaseURL(r) - - if debug { - utils.SafeLogf("[DEBUG] Base URL set to %s\n", baseUrl) - } - - var content strings.Builder - content.WriteString("#EXTM3U\n") - - // Retrieve the streams from the database using channels - streamChan := db.GetStreams() - for stream := range streamChan { - if len(stream.URLs) == 0 { - continue - } - - if debug { - utils.SafeLogf("[DEBUG] Processing stream with TVG ID: %s\n", stream.TvgID) - } - - extInfTags := []string{ - "#EXTINF:-1", - } - - if len(stream.TvgID) > 0 { - extInfTags = append(extInfTags, fmt.Sprintf("tvg-id=\"%s\"", stream.TvgID)) - } - - if len(stream.TvgChNo) > 0 { - extInfTags = append(extInfTags, fmt.Sprintf("tvg-chno=\"%s\"", stream.TvgChNo)) - } - - if len(stream.LogoURL) > 0 { - extInfTags = append(extInfTags, fmt.Sprintf("tvg-logo=\"%s\"", stream.LogoURL)) - } - - extInfTags = append(extInfTags, fmt.Sprintf("tvg-name=\"%s\"", stream.Title)) - extInfTags = append(extInfTags, fmt.Sprintf("group-title=\"%s\"", stream.Group)) - - content.WriteString(fmt.Sprintf("%s,%s\n", strings.Join(extInfTags, " "), stream.Title)) - content.WriteString(GenerateStreamURL(baseUrl, stream)) - } - - if debug { - utils.SafeLogln("[DEBUG] Finished generating M3U content") - } - - // Update cache - M3uCache.Lock() - M3uCache.data = content.String() - M3uCache.Revalidating = false - M3uCache.Unlock() - - return content.String() -} - -func ClearCache() { - debug := isDebugMode() - - M3uCache.Lock() - - if debug { - utils.SafeLogln("[DEBUG] Clearing memory and disk M3U cache.") - } - M3uCache.data = "" - if err := DeleteCacheFile(); err != nil { - if debug { - utils.SafeLogf("[DEBUG] Cache file deletion failed: %v\n", err) - } - } - - M3uCache.Unlock() -} - -func Handler(w http.ResponseWriter, r *http.Request) { - db, err := database.InitializeDb() - if err != nil { - utils.SafeLogf("Error initializing Redis database: %v", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - debug := isDebugMode() - - if debug { - utils.SafeLogln("[DEBUG] Generating M3U content") - } - - // Set response headers - w.Header().Set("Content-Type", "text/plain") - w.Header().Set("Access-Control-Allow-Origin", "*") - - M3uCache.Lock() - cacheData := M3uCache.data - M3uCache.Unlock() - - if cacheData == "#EXTM3U\n" || cacheData == "" { - // Check the file-based cache - if fileData, err := ReadCacheFromFile(); err == nil { - cacheData = fileData - M3uCache.Lock() - M3uCache.data = fileData // update in-memory cache - M3uCache.Unlock() - } - } - - // serve old cache and regenerate in the background - if cacheData != "#EXTM3U\n" && cacheData != "" { - if debug { - utils.SafeLogln("[DEBUG] Serving old cache and regenerating in background") - } - if _, err := w.Write([]byte(cacheData)); err != nil { - utils.SafeLogf("[ERROR] Failed to write response: %v\n", err) - } - - InitCache(db) - - return - } - - // If no valid cache, generate content and update cache - content := GenerateAndCacheM3UContent(db, r) - go func() { - if err := WriteCacheToFile(content); err != nil { - utils.SafeLogf("[ERROR] Failed to write cache to file: %v\n", err) - } - }() - - if _, err := w.Write([]byte(content)); err != nil { - utils.SafeLogf("[ERROR] Failed to write response: %v\n", err) - } -} - -func ReadCacheFromFile() (string, error) { - data, err := os.ReadFile(cacheFilePath) - if err != nil { - return "", err - } - return string(data), nil -} - -func WriteCacheToFile(content string) error { - return os.WriteFile(cacheFilePath, []byte(content), 0644) -} - -func DeleteCacheFile() error { - return os.Remove(cacheFilePath) -} diff --git a/m3u/m3u_test.go b/m3u/m3u_test.go deleted file mode 100644 index 5d8d1ad7..00000000 --- a/m3u/m3u_test.go +++ /dev/null @@ -1,191 +0,0 @@ -package m3u - -import ( - "fmt" - "net/http" - "net/http/httptest" - "os" - "testing" - - "m3u-stream-merger/database" -) - -func TestGenerateM3UContent(t *testing.T) { - // Define a sample stream for testing - stream := database.StreamInfo{ - Slug: "test-stream", - TvgID: "1", - Title: "TestStream", - LogoURL: "http://example.com/logo.png", - Group: "TestGroup", - URLs: map[int]string{0: "http://example.com/stream"}, - } - - os.Setenv("REDIS_DB", "2") - - db, err := database.InitializeDb() - if err != nil { - t.Errorf("InitializeDb returned error: %v", err) - } - - err = db.ClearDb() - if err != nil { - t.Errorf("ClearDb returned error: %v", err) - } - - err = db.SaveToDb([]*database.StreamInfo{&stream}) - if err != nil { - t.Fatal(err) - } - - // Create a new HTTP request - req, err := http.NewRequest("GET", "/generate", nil) - if err != nil { - t.Fatal(err) - } - - // Create a ResponseRecorder to record the response - rr := httptest.NewRecorder() - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - Handler(w, r) - }) - - // Call the ServeHTTP method of the handler to execute the test - handler.ServeHTTP(rr, req) - - // Check the status code - if status := rr.Code; status != http.StatusOK { - t.Errorf("handler returned wrong status code: got %v want %v", - status, http.StatusOK) - } - - // Check the Content-Type header - expectedContentType := "text/plain" - if contentType := rr.Header().Get("Content-Type"); contentType != expectedContentType { - t.Errorf("handler returned unexpected Content-Type: got %v want %v", - contentType, expectedContentType) - } - - // Check the generated M3U content - expectedContent := fmt.Sprintf(`#EXTM3U -#EXTINF:-1 tvg-id="1" tvg-logo="http://example.com/logo.png" tvg-name="TestStream" group-title="TestGroup",TestStream -%s`, GenerateStreamURL("http://", stream)) - if rr.Body.String() != expectedContent { - t.Errorf("handler returned unexpected body: got %v want %v", - rr.Body.String(), expectedContent) - } -} - -func TestParseM3UFromURL(t *testing.T) { - testM3UContent := ` -#EXTM3U -#EXTINF:-1 channelID="x-ID.bcc1" tvg-chno="0.0" tvg-id="bbc1" tvg-name="BBC One" group-title="UK",BBC One -#EXTVLCOPT:http-user-agent=HbbTV/1.6.1 -http://example.com/bbc1 -#EXTINF:-1 channelID="x-ID.bcc2" tvg-chno="0.0" tvg-id="bbc2" tvg-name="BBC Two" group-title="UK",BBC Two -#EXTVLCOPT:http-user-agent=HbbTV/1.6.1 -http://example.com/bbc2 -#EXTINF:-1 channelID="x-ID.cnn" tvg-chno="0.0" tvg-id="cnn" tvg-name="CNN International" group-title="News",CNN International -#EXTVLCOPT:http-user-agent=HbbTV/1.6.1 -http://example.com/cnn -#EXTVLCOPT:logo=http://example.com/bbc_logo.png -#EXTINF:-1 channelID="x-ID.fox" tvg-chno="0.0" tvg-name="FOX" group-title="Entertainment",FOX -http://example.com/fox -` - // Create a mock HTTP server - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/octet-stream") - fmt.Fprintln(w, testM3UContent) - })) - defer mockServer.Close() - - // Create test file - mockFile := "test-playlist.m3u" - err := os.WriteFile(mockFile, []byte(testM3UContent), 0644) - if err != nil { - t.Errorf("WriteFile returned error: %v", err) - } - - os.Setenv("REDIS_DB", "3") - db, err := database.InitializeDb() - if err != nil { - t.Errorf("InitializeDb returned error: %v", err) - } - - err = db.ClearDb() - if err != nil { - t.Errorf("ClearDb returned error: %v", err) - } - - parser := InitializeParser() - - // Test the parseM3UFromURL function with the mock server URL - err = parser.ParseURL(mockServer.URL, 0) - if err != nil { - t.Errorf("Error parsing M3U from URL: %v", err) - } - - // Test the parseM3UFromURL function with the mock server URL - err = parser.ParseURL(fmt.Sprintf("file://%s", mockFile), 1) - if err != nil { - t.Errorf("Error parsing M3U from URL: %v", err) - } - - err = db.SaveToDb(parser.GetStreams()) - if err != nil { - t.Errorf("Error store to db: %v", err) - } - - // Verify expected values - expectedStreams := []database.StreamInfo{ - {Slug: "bbc-one", Title: "BBC One", TvgChNo: "0.0", TvgID: "bbc1", Group: "UK", URLs: map[int]string{0: "http://example.com/bbc1", 1: "http://example.com/bbc1"}}, - {Slug: "bbc-two", Title: "BBC Two", TvgChNo: "0.0", TvgID: "bbc2", Group: "UK", URLs: map[int]string{0: "http://example.com/bbc2", 1: "http://example.com/bbc2"}}, - {Slug: "cnn-international", Title: "CNN International", TvgChNo: "0.0", TvgID: "cnn", Group: "News", URLs: map[int]string{0: "http://example.com/cnn", 1: "http://example.com/cnn"}}, - {Slug: "fox", Title: "FOX", TvgChNo: "0.0", Group: "Entertainment", URLs: map[int]string{0: "http://example.com/fox", 1: "http://example.com/fox"}}, - } - - streamChan := db.GetStreams() - - storedStreams := []database.StreamInfo{} - for stream := range streamChan { - storedStreams = append(storedStreams, stream) - } - - // Compare the retrieved streams with the expected streams - if len(storedStreams) != len(expectedStreams) { - t.Fatalf("Expected %d streams, but got %d", len(expectedStreams), len(storedStreams)) - } - - // Create a map to store expected streams for easier comparison - expectedMap := make(map[string]database.StreamInfo) - for _, expected := range expectedStreams { - expectedMap[expected.Title] = expected - } - - for _, stored := range storedStreams { - expected, ok := expectedMap[stored.Title] - if !ok { - t.Errorf("Unexpected stream with Title: %s", stored.Title) - continue - } - if !streamInfoEqual(stored, expected) { - t.Errorf("Stream with Title %s does not match expected content", stored.Title) - t.Errorf("Stored: %#v, Expected: %#v", stored, expected) - } - } -} - -// streamInfoEqual checks if two StreamInfo objects are equal. -func streamInfoEqual(a, b database.StreamInfo) bool { - if a.Slug != b.Slug || a.TvgID != b.TvgID || a.TvgChNo != b.TvgChNo || a.Title != b.Title || a.Group != b.Group || a.LogoURL != b.LogoURL || len(a.URLs) != len(b.URLs) { - return false - } - - for i, url := range a.URLs { - if url != b.URLs[i] { - return false - } - } - - return true -} diff --git a/m3u/middlewares.go b/m3u/middlewares.go deleted file mode 100644 index 9ed0924b..00000000 --- a/m3u/middlewares.go +++ /dev/null @@ -1,47 +0,0 @@ -package m3u - -import ( - "m3u-stream-merger/utils" - "os" - "regexp" - "strings" -) - -func generalParser(value string) string { - if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) { - value = strings.Trim(value, `"`) - } - - return value -} - -func tvgNameParser(value string) string { - substrFilter := os.Getenv("TITLE_SUBSTR_FILTER") - // Apply character filter - if substrFilter != "" { - re, err := regexp.Compile(substrFilter) - if err != nil { - utils.SafeLogf("Error compiling character filter regex: %v\n", err) - } else { - value = re.ReplaceAllString(value, "") - } - } - - return generalParser(value) -} - -func tvgIdParser(value string) string { - return generalParser(value) -} - -func tvgChNoParser(value string) string { - return generalParser(value) -} - -func groupTitleParser(value string) string { - return generalParser(value) -} - -func tvgLogoParser(value string) string { - return generalParser(value) -} diff --git a/m3u/parser.go b/m3u/parser.go deleted file mode 100644 index 5f77f710..00000000 --- a/m3u/parser.go +++ /dev/null @@ -1,259 +0,0 @@ -package m3u - -import ( - "bufio" - "bytes" - "fmt" - "maps" - "os" - "regexp" - "slices" - "strconv" - "strings" - "sync" - "time" - - "m3u-stream-merger/database" - "m3u-stream-merger/utils" - - "github.com/gosimple/slug" -) - -type Parser struct { - sync.Mutex - Streams map[string]*database.StreamInfo -} - -func InitializeParser() *Parser { - return &Parser{ - Streams: map[string]*database.StreamInfo{}, - } -} - -func parseLine(line string, nextLine string, m3uIndex int) database.StreamInfo { - debug := os.Getenv("DEBUG") == "true" - if debug { - utils.SafeLogf("[DEBUG] Parsing line: %s\n", line) - utils.SafeLogf("[DEBUG] Next line: %s\n", nextLine) - utils.SafeLogf("[DEBUG] M3U index: %d\n", m3uIndex) - } - - currentStream := database.StreamInfo{} - currentStream.URLs = map[int]string{m3uIndex: strings.TrimSpace(nextLine)} - - lineWithoutPairs := line - - // Define a regular expression to capture key-value pairs - regex := regexp.MustCompile(`([a-zA-Z0-9_-]+)="([^"]+)"`) - - // Find all key-value pairs in the line - matches := regex.FindAllStringSubmatch(line, -1) - - for _, match := range matches { - key := strings.TrimSpace(match[1]) - value := strings.TrimSpace(match[2]) - - if debug { - utils.SafeLogf("[DEBUG] Processing attribute: %s=%s\n", key, value) - } - - switch strings.ToLower(key) { - case "tvg-id": - currentStream.TvgID = tvgIdParser(value) - case "tvg-chno": - currentStream.TvgChNo = tvgChNoParser(value) - case "tvg-name": - currentStream.Title = tvgNameParser(value) - case "group-title": - currentStream.Group = groupTitleParser(value) - case "tvg-logo": - currentStream.LogoURL = tvgLogoParser(value) - default: - if debug { - utils.SafeLogf("[DEBUG] Uncaught attribute: %s=%s\n", key, value) - } - } - - lineWithoutPairs = strings.Replace(lineWithoutPairs, match[0], "", 1) - } - - lineCommaSplit := strings.SplitN(lineWithoutPairs, ",", 2) - - if len(lineCommaSplit) > 1 { - if debug { - utils.SafeLogf("[DEBUG] Line comma split detected, title: %s\n", strings.TrimSpace(lineCommaSplit[1])) - } - currentStream.Title = tvgNameParser(strings.TrimSpace(lineCommaSplit[1])) - } - - currentStream.Slug = slug.Make(currentStream.Title) - - if debug { - utils.SafeLogf("[DEBUG] Generated slug: %s\n", currentStream.Slug) - } - - return currentStream -} - -func (instance *Parser) GetStreams() []*database.StreamInfo { - storeArray := []*database.StreamInfo{} - storeValues := maps.Values(instance.Streams) - if storeValues != nil { - storeArray = slices.Collect(storeValues) - } - - return storeArray -} - -func (instance *Parser) ParseURL(m3uURL string, m3uIndex int) error { - debug := os.Getenv("DEBUG") == "true" - - maxRetries := 10 - retryWait := 0 - - var err error - maxRetriesStr, maxRetriesExists := os.LookupEnv("MAX_RETRIES") - if maxRetriesExists { - maxRetries, err = strconv.Atoi(maxRetriesStr) - if err != nil { - maxRetries = 10 - } - } - - retryWaitStr, retryWaitExists := os.LookupEnv("RETRY_WAIT") - if retryWaitExists { - retryWait, err = strconv.Atoi(retryWaitStr) - if err != nil { - retryWait = 0 - } - } - - if debug { - utils.SafeLogf("[DEBUG] Max retries set to %d\n", maxRetries) - } - - var buffer bytes.Buffer - for i := 0; i <= maxRetries; i++ { - if i > 0 && retryWait > 0 { - utils.SafeLogf("Retrying in %d secs...\n", retryWait) - time.Sleep(time.Duration(retryWait) * time.Second) - } - - if debug { - utils.SafeLogf("[DEBUG] Attempt %d to download M3U\n", i+1) - } - err := downloadM3UToBuffer(m3uURL, &buffer) - if err != nil { - utils.SafeLogf("downloadM3UToBuffer error: %v\n", err) - continue - } - - utils.SafeLogln("Parsing downloaded M3U file.") - scanner := bufio.NewScanner(&buffer) - var wg sync.WaitGroup - - parserWorkers := os.Getenv("PARSER_WORKERS") - if strings.TrimSpace(parserWorkers) == "" { - parserWorkers = "5" - } - - if debug { - utils.SafeLogf("[DEBUG] Using %s parser workers\n", parserWorkers) - } - - streamInfoCh := make(chan database.StreamInfo) - errCh := make(chan error) - numWorkers, err := strconv.Atoi(parserWorkers) - if err != nil { - numWorkers = 5 - } - - for w := 0; w < numWorkers; w++ { - wg.Add(1) - go func() { - defer wg.Done() - for streamInfo := range streamInfoCh { - if debug { - utils.SafeLogf("[DEBUG] Worker processing stream info: %s\n", streamInfo.Slug) - } - - instance.Lock() - _, ok := instance.Streams[streamInfo.Title] - if !ok { - instance.Streams[streamInfo.Title] = &streamInfo - } else { - if instance.Streams[streamInfo.Title].URLs == nil { - instance.Streams[streamInfo.Title].URLs = map[int]string{} - } - - if streamInfo.URLs != nil { - maps.Copy(instance.Streams[streamInfo.Title].URLs, streamInfo.URLs) - } - } - instance.Unlock() - } - }() - } - - go func() { - for err := range errCh { - utils.SafeLogf("M3U Parser error: %v\n", err) - } - }() - - for scanner.Scan() { - line := scanner.Text() - - if debug { - utils.SafeLogf("[DEBUG] Scanning line: %s\n", line) - } - - if strings.HasPrefix(line, "#EXTINF:") { - if scanner.Scan() { - nextLine := scanner.Text() - // skip all other #EXT tags - for strings.HasPrefix(nextLine, "#") { - if scanner.Scan() { - nextLine = scanner.Text() - } else { - break - } - } - - if debug { - utils.SafeLogf("[DEBUG] Found next line for EXTINF: %s\n", nextLine) - } - - streamInfo := parseLine(line, nextLine, m3uIndex) - - if checkFilter(streamInfo) { - utils.SafeLogf("Stream Info: %v\n", streamInfo) - streamInfoCh <- streamInfo - } else { - if debug { - utils.SafeLogf("[DEBUG] Skipping due to filter: %s\n", streamInfo.Title) - } - } - } - } - } - - close(streamInfoCh) - wg.Wait() - close(errCh) - - if err := scanner.Err(); err != nil { - return fmt.Errorf("scanner error: %v", err) - } - - buffer.Reset() - - if debug { - utils.SafeLogln("[DEBUG] Buffer reset and memory freed") - } - - return nil - } - - return fmt.Errorf("Max retries reached without success. Failed to fetch %s\n", m3uURL) -} diff --git a/main.go b/main.go index 828406d4..44a0cf8a 100644 --- a/main.go +++ b/main.go @@ -3,9 +3,8 @@ package main import ( "context" "fmt" - "m3u-stream-merger/database" - "m3u-stream-merger/m3u" - "m3u-stream-merger/proxy" + "m3u-stream-merger/handlers" + "m3u-stream-merger/store" "m3u-stream-merger/updater" "m3u-stream-merger/utils" "net/http" @@ -18,14 +17,10 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - utils.SafeLogln("Checking database connection...") - db, err := database.InitializeDb() - if err != nil { - utils.SafeLogFatalf("Error initializing Redis database: %v", err) - } + cm := store.NewConcurrencyManager() utils.SafeLogln("Starting updater...") - _, err = updater.Initialize(ctx) + _, err := updater.Initialize(ctx) if err != nil { utils.SafeLogFatalf("Error initializing updater: %v", err) } @@ -39,19 +34,13 @@ func main() { } } - utils.SafeLogln("Clearing stale concurrency data from database...") - err = db.ClearConcurrencies() - if err != nil { - utils.SafeLogFatalf("Error clearing concurrency database: %v", err) - } - utils.SafeLogln("Setting up HTTP handlers...") // HTTP handlers http.HandleFunc("/playlist.m3u", func(w http.ResponseWriter, r *http.Request) { - m3u.Handler(w, r) + handlers.M3UHandler(w, r) }) http.HandleFunc("/proxy/", func(w http.ResponseWriter, r *http.Request) { - proxy.Handler(w, r) + handlers.StreamHandler(w, r, cm) }) // Start the server diff --git a/proxy/load_balancer.go b/proxy/load_balancer.go new file mode 100644 index 00000000..f3908080 --- /dev/null +++ b/proxy/load_balancer.go @@ -0,0 +1,98 @@ +package proxy + +import ( + "fmt" + "m3u-stream-merger/store" + "m3u-stream-merger/utils" + "net/http" + "os" + "slices" + "sort" + "strconv" + "strings" +) + +type StreamInstance struct { + Info store.StreamInfo + Cm *store.ConcurrencyManager +} + +func NewStreamInstance(streamUrl string, cm *store.ConcurrencyManager) (*StreamInstance, error) { + stream, err := store.GetStreamBySlug(streamUrl) + if err != nil { + return nil, err + } + + return &StreamInstance{ + Info: stream, + Cm: cm, + }, nil +} + +func (instance *StreamInstance) LoadBalancer(previous *[]int, method string) (*http.Response, string, int, error) { + debug := os.Getenv("DEBUG") == "true" + + m3uIndexes := utils.GetM3UIndexes() + + sort.Slice(m3uIndexes, func(i, j int) bool { + return instance.Cm.ConcurrencyPriorityValue(i) > instance.Cm.ConcurrencyPriorityValue(j) + }) + + maxLapsString := os.Getenv("MAX_RETRIES") + maxLaps, err := strconv.Atoi(strings.TrimSpace(maxLapsString)) + if err != nil || maxLaps < 0 { + maxLaps = 5 + } + + lap := 0 + + for lap < maxLaps || maxLaps == 0 { + if debug { + utils.SafeLogf("[DEBUG] Stream attempt %d out of %d\n", lap+1, maxLaps) + } + allSkipped := true // Assume all URLs might be skipped + + for _, index := range m3uIndexes { + if slices.Contains(*previous, index) { + utils.SafeLogf("Skipping M3U_%d: marked as previous stream\n", index+1) + continue + } + + url, ok := instance.Info.URLs[index] + if !ok { + utils.SafeLogf("Channel not found from M3U_%d: %s\n", index+1, instance.Info.Title) + continue + } + + if instance.Cm.CheckConcurrency(index) { + utils.SafeLogf("Concurrency limit reached for M3U_%d: %s\n", index+1, url) + continue + } + + allSkipped = false // At least one URL is not skipped + + resp, err := utils.CustomHttpRequest(method, url) + if err == nil { + if debug { + utils.SafeLogf("[DEBUG] Successfully fetched stream from %s\n", url) + } + return resp, url, index, nil + } + utils.SafeLogf("Error fetching stream: %s\n", err.Error()) + if debug { + utils.SafeLogf("[DEBUG] Error fetching stream from %s: %s\n", url, err.Error()) + } + } + + if allSkipped { + if debug { + utils.SafeLogf("[DEBUG] All streams skipped in lap %d\n", lap) + } + *previous = []int{} + } + + lap++ + } + + return nil, "", -1, fmt.Errorf("Error fetching stream. Exhausted all streams.") +} diff --git a/proxy/proxy_stream.go b/proxy/proxy_stream.go new file mode 100644 index 00000000..b024dca7 --- /dev/null +++ b/proxy/proxy_stream.go @@ -0,0 +1,184 @@ +package proxy + +import ( + "bufio" + "context" + "io" + "m3u-stream-merger/utils" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" +) + +func (instance *StreamInstance) ProxyStream(ctx context.Context, m3uIndex int, resp *http.Response, r *http.Request, w http.ResponseWriter, statusChan chan int) { + debug := os.Getenv("DEBUG") == "true" + bufferMbInt, err := strconv.Atoi(os.Getenv("BUFFER_MB")) + if err != nil || bufferMbInt < 0 { + bufferMbInt = 0 + } + buffer := make([]byte, 1024) + if bufferMbInt > 0 { + buffer = make([]byte, bufferMbInt*1024*1024) + } + + if r.Method != http.MethodGet || utils.EOFIsExpected(resp) { + scanner := bufio.NewScanner(resp.Body) + base, err := url.Parse(resp.Request.URL.String()) + if err != nil { + utils.SafeLogf("Invalid base URL for M3U8 stream: %v", err) + statusChan <- 4 + return + } + + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "#") { + _, err := w.Write([]byte(line + "\n")) + if err != nil { + utils.SafeLogf("Failed to write line to response: %v", err) + statusChan <- 4 + return + } + } else if strings.TrimSpace(line) != "" { + u, err := url.Parse(line) + if err != nil { + utils.SafeLogf("Failed to parse M3U8 URL in line: %v", err) + _, err := w.Write([]byte(line + "\n")) + if err != nil { + utils.SafeLogf("Failed to write line to response: %v", err) + statusChan <- 4 + return + } + continue + } + + if !u.IsAbs() { + u = base.ResolveReference(u) + } + + _, err = w.Write([]byte(u.String() + "\n")) + if err != nil { + utils.SafeLogf("Failed to write URL to response: %v", err) + statusChan <- 4 + return + } + } + } + + statusChan <- 4 + return + } + + instance.Cm.UpdateConcurrency(m3uIndex, true) + defer instance.Cm.UpdateConcurrency(m3uIndex, false) + + defer func() { + buffer = nil + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + }() + + timeoutSecond, err := strconv.Atoi(os.Getenv("STREAM_TIMEOUT")) + if err != nil || timeoutSecond < 0 { + timeoutSecond = 3 + } + + timeoutDuration := time.Duration(timeoutSecond) * time.Second + if timeoutSecond == 0 { + timeoutDuration = time.Minute + } + timer := time.NewTimer(timeoutDuration) + defer timer.Stop() + + // Backoff settings + initialBackoff := 200 * time.Millisecond + maxBackoff := time.Duration(timeoutSecond-1) * time.Second + currentBackoff := initialBackoff + + returnStatus := 0 + + for { + select { + case <-ctx.Done(): // handle context cancellation + utils.SafeLogf("Context canceled for stream: %s\n", r.RemoteAddr) + statusChan <- 0 + return + case <-timer.C: + utils.SafeLogf("Timeout reached while trying to stream: %s\n", r.RemoteAddr) + statusChan <- returnStatus + return + default: + n, err := resp.Body.Read(buffer) + if err != nil { + if err == io.EOF { + utils.SafeLogf("Stream ended (EOF reached): %s\n", r.RemoteAddr) + if utils.EOFIsExpected(resp) || timeoutSecond == 0 { + statusChan <- 2 + return + } + + returnStatus = 2 + utils.SafeLogf("Retrying same stream until timeout (%d seconds) is reached...\n", timeoutSecond) + if debug { + utils.SafeLogf("[DEBUG] Retrying same stream with backoff of %v...\n", currentBackoff) + } + + time.Sleep(currentBackoff) + currentBackoff *= 2 + if currentBackoff > maxBackoff { + currentBackoff = maxBackoff + } + + continue + } + + utils.SafeLogf("Error reading stream: %s\n", err.Error()) + + returnStatus = 1 + + if timeoutSecond == 0 { + statusChan <- 1 + return + } + + if debug { + utils.SafeLogf("[DEBUG] Retrying same stream with backoff of %v...\n", currentBackoff) + } + + time.Sleep(currentBackoff) + currentBackoff *= 2 + if currentBackoff > maxBackoff { + currentBackoff = maxBackoff + } + + continue + } + + if _, err := w.Write(buffer[:n]); err != nil { + utils.SafeLogf("Error writing to response: %s\n", err.Error()) + statusChan <- 0 + return + } + + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + + // Reset the timer on each successful write and backoff + if !timer.Stop() { + select { + case <-timer.C: // drain the channel to avoid blocking + default: + } + } + timer.Reset(timeoutDuration) + + // Reset the backoff duration after successful read/write + currentBackoff = initialBackoff + } + } +} diff --git a/proxy/stream_handler.go b/proxy/stream_handler.go deleted file mode 100644 index cf544d4a..00000000 --- a/proxy/stream_handler.go +++ /dev/null @@ -1,370 +0,0 @@ -package proxy - -import ( - "bufio" - "context" - "fmt" - "io" - "m3u-stream-merger/database" - "m3u-stream-merger/utils" - "net/http" - "net/url" - "os" - "path" - "slices" - "sort" - "strconv" - "strings" - "time" -) - -type StreamInstance struct { - Database *database.Instance - Info database.StreamInfo -} - -func InitializeStream(streamUrl string) (*StreamInstance, error) { - initDb, err := database.InitializeDb() - if err != nil { - utils.SafeLogf("Error initializing Redis database: %v", err) - return nil, err - } - - stream, err := initDb.GetStreamBySlug(streamUrl) - if err != nil { - return nil, err - } - - return &StreamInstance{ - Database: initDb, - Info: stream, - }, nil -} - -func (instance *StreamInstance) LoadBalancer(previous *[]int, method string) (*http.Response, string, int, error) { - debug := os.Getenv("DEBUG") == "true" - - m3uIndexes := utils.GetM3UIndexes() - - sort.Slice(m3uIndexes, func(i, j int) bool { - return instance.Database.ConcurrencyPriorityValue(i) > instance.Database.ConcurrencyPriorityValue(j) - }) - - maxLapsString := os.Getenv("MAX_RETRIES") - maxLaps, err := strconv.Atoi(strings.TrimSpace(maxLapsString)) - if err != nil || maxLaps < 0 { - maxLaps = 5 - } - - lap := 0 - - for lap < maxLaps || maxLaps == 0 { - if debug { - utils.SafeLogf("[DEBUG] Stream attempt %d out of %d\n", lap+1, maxLaps) - } - allSkipped := true // Assume all URLs might be skipped - - for _, index := range m3uIndexes { - if slices.Contains(*previous, index) { - utils.SafeLogf("Skipping M3U_%d: marked as previous stream\n", index+1) - continue - } - - url, ok := instance.Info.URLs[index] - if !ok { - utils.SafeLogf("Channel not found from M3U_%d: %s\n", index+1, instance.Info.Title) - continue - } - - if instance.Database.CheckConcurrency(index) { - utils.SafeLogf("Concurrency limit reached for M3U_%d: %s\n", index+1, url) - continue - } - - allSkipped = false // At least one URL is not skipped - - resp, err := utils.CustomHttpRequest(method, url) - if err == nil { - if debug { - utils.SafeLogf("[DEBUG] Successfully fetched stream from %s\n", url) - } - return resp, url, index, nil - } - utils.SafeLogf("Error fetching stream: %s\n", err.Error()) - if debug { - utils.SafeLogf("[DEBUG] Error fetching stream from %s: %s\n", url, err.Error()) - } - } - - if allSkipped { - if debug { - utils.SafeLogf("[DEBUG] All streams skipped in lap %d\n", lap) - } - *previous = []int{} - } - - lap++ - } - - return nil, "", -1, fmt.Errorf("Error fetching stream. Exhausted all streams.") -} - -func (instance *StreamInstance) ProxyStream(ctx context.Context, m3uIndex int, resp *http.Response, r *http.Request, w http.ResponseWriter, statusChan chan int) { - debug := os.Getenv("DEBUG") == "true" - bufferMbInt, err := strconv.Atoi(os.Getenv("BUFFER_MB")) - if err != nil || bufferMbInt < 0 { - bufferMbInt = 0 - } - buffer := make([]byte, 1024) - if bufferMbInt > 0 { - buffer = make([]byte, bufferMbInt*1024*1024) - } - - if r.Method != http.MethodGet || utils.EOFIsExpected(resp) { - scanner := bufio.NewScanner(resp.Body) - base, err := url.Parse(resp.Request.URL.String()) - if err != nil { - utils.SafeLogf("Invalid base URL for M3U8 stream: %v", err) - statusChan <- 4 - return - } - - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "#") { - _, err := w.Write([]byte(line + "\n")) - if err != nil { - utils.SafeLogf("Failed to write line to response: %v", err) - statusChan <- 4 - return - } - } else if strings.TrimSpace(line) != "" { - u, err := url.Parse(line) - if err != nil { - utils.SafeLogf("Failed to parse M3U8 URL in line: %v", err) - _, err := w.Write([]byte(line + "\n")) - if err != nil { - utils.SafeLogf("Failed to write line to response: %v", err) - statusChan <- 4 - return - } - continue - } - - if !u.IsAbs() { - u = base.ResolveReference(u) - } - - _, err = w.Write([]byte(u.String() + "\n")) - if err != nil { - utils.SafeLogf("Failed to write URL to response: %v", err) - statusChan <- 4 - return - } - } - } - - statusChan <- 4 - return - } - - instance.Database.UpdateConcurrency(m3uIndex, true) - defer instance.Database.UpdateConcurrency(m3uIndex, false) - - defer func() { - buffer = nil - if flusher, ok := w.(http.Flusher); ok { - flusher.Flush() - } - }() - - timeoutSecond, err := strconv.Atoi(os.Getenv("STREAM_TIMEOUT")) - if err != nil || timeoutSecond < 0 { - timeoutSecond = 3 - } - - timeoutDuration := time.Duration(timeoutSecond) * time.Second - if timeoutSecond == 0 { - timeoutDuration = time.Minute - } - timer := time.NewTimer(timeoutDuration) - defer timer.Stop() - - // Backoff settings - initialBackoff := 200 * time.Millisecond - maxBackoff := time.Duration(timeoutSecond-1) * time.Second - currentBackoff := initialBackoff - - returnStatus := 0 - - for { - select { - case <-ctx.Done(): // handle context cancellation - utils.SafeLogf("Context canceled for stream: %s\n", r.RemoteAddr) - statusChan <- 0 - return - case <-timer.C: - utils.SafeLogf("Timeout reached while trying to stream: %s\n", r.RemoteAddr) - statusChan <- returnStatus - return - default: - n, err := resp.Body.Read(buffer) - if err != nil { - if err == io.EOF { - utils.SafeLogf("Stream ended (EOF reached): %s\n", r.RemoteAddr) - if utils.EOFIsExpected(resp) || timeoutSecond == 0 { - statusChan <- 2 - return - } - - returnStatus = 2 - utils.SafeLogf("Retrying same stream until timeout (%d seconds) is reached...\n", timeoutSecond) - if debug { - utils.SafeLogf("[DEBUG] Retrying same stream with backoff of %v...\n", currentBackoff) - } - - time.Sleep(currentBackoff) - currentBackoff *= 2 - if currentBackoff > maxBackoff { - currentBackoff = maxBackoff - } - - continue - } - - utils.SafeLogf("Error reading stream: %s\n", err.Error()) - - returnStatus = 1 - - if timeoutSecond == 0 { - statusChan <- 1 - return - } - - if debug { - utils.SafeLogf("[DEBUG] Retrying same stream with backoff of %v...\n", currentBackoff) - } - - time.Sleep(currentBackoff) - currentBackoff *= 2 - if currentBackoff > maxBackoff { - currentBackoff = maxBackoff - } - - continue - } - - if _, err := w.Write(buffer[:n]); err != nil { - utils.SafeLogf("Error writing to response: %s\n", err.Error()) - statusChan <- 0 - return - } - - if flusher, ok := w.(http.Flusher); ok { - flusher.Flush() - } - - // Reset the timer on each successful write and backoff - if !timer.Stop() { - select { - case <-timer.C: // drain the channel to avoid blocking - default: - } - } - timer.Reset(timeoutDuration) - - // Reset the backoff duration after successful read/write - currentBackoff = initialBackoff - } - } -} - -func Handler(w http.ResponseWriter, r *http.Request) { - debug := os.Getenv("DEBUG") == "true" - - ctx, cancel := context.WithCancel(r.Context()) - defer cancel() - - utils.SafeLogf("Received request from %s for URL: %s\n", r.RemoteAddr, r.URL.Path) - - streamUrl := strings.Split(path.Base(r.URL.Path), ".")[0] - if streamUrl == "" { - utils.SafeLogf("Invalid m3uID for request from %s: %s\n", r.RemoteAddr, r.URL.Path) - http.NotFound(w, r) - return - } - - stream, err := InitializeStream(strings.TrimPrefix(streamUrl, "/")) - if err != nil { - utils.SafeLogf("Error retrieving stream for slug %s: %v\n", streamUrl, err) - http.NotFound(w, r) - return - } - - var selectedIndex int - var selectedUrl string - - testedIndexes := []int{} - firstWrite := true - - var resp *http.Response - - for { - select { - case <-ctx.Done(): - utils.SafeLogf("Client disconnected: %s\n", r.RemoteAddr) - return - default: - resp, selectedUrl, selectedIndex, err = stream.LoadBalancer(&testedIndexes, r.Method) - if err != nil { - utils.SafeLogf("Error reloading stream for %s: %v\n", streamUrl, err) - return - } - - // HTTP header initialization - if firstWrite { - for k, v := range resp.Header { - if strings.ToLower(k) == "content-length" { - continue - } - - for _, val := range v { - w.Header().Set(k, val) - } - } - w.WriteHeader(resp.StatusCode) - - if debug { - utils.SafeLogf("[DEBUG] Headers set for response: %v\n", w.Header()) - } - firstWrite = false - } - - exitStatus := make(chan int) - - utils.SafeLogf("Proxying %s to %s\n", r.RemoteAddr, selectedUrl) - go stream.ProxyStream(ctx, selectedIndex, resp, r, w, exitStatus) - testedIndexes = append(testedIndexes, selectedIndex) - - streamExitCode := <-exitStatus - utils.SafeLogf("Exit code %d received from %s\n", streamExitCode, selectedUrl) - - if streamExitCode == 2 && utils.EOFIsExpected(resp) { - utils.SafeLogf("Successfully proxied playlist: %s\n", r.RemoteAddr) - cancel() - } else if streamExitCode == 1 || streamExitCode == 2 { - // Retry on server-side connection errors - utils.SafeLogf("Retrying other servers...\n") - } else if streamExitCode == 4 { - utils.SafeLogf("Finished handling %s request: %s\n", r.Method, r.RemoteAddr) - cancel() - } else { - // Consider client-side connection errors as complete closure - utils.SafeLogf("Client has closed the stream: %s\n", r.RemoteAddr) - cancel() - } - - resp.Body.Close() - } - } -} diff --git a/store/cache.go b/store/cache.go new file mode 100644 index 00000000..c257846f --- /dev/null +++ b/store/cache.go @@ -0,0 +1,166 @@ +package store + +import ( + "fmt" + "m3u-stream-merger/utils" + "net/http" + "os" + "strings" + "sync" + "time" +) + +type Cache struct { + sync.RWMutex + Exists bool + Revalidating bool +} + +var M3uCache = &Cache{} + +const cacheFilePath = "/m3u-proxy/cache.m3u" + +func isDebugMode() bool { + return os.Getenv("DEBUG") == "true" +} + +func RevalidatingGetM3U(r *http.Request) string { + M3uCache.RLock() + if M3uCache.Exists { + if !M3uCache.Revalidating { + M3uCache.Lock() + M3uCache.Revalidating = true + M3uCache.Unlock() + + go generateM3UContent(r) + } + + M3uCache.RUnlock() + return readCacheFromFile() + } + M3uCache.RUnlock() + + return generateM3UContent(r) +} + +func generateM3UContent(r *http.Request) string { + M3uCache.RLock() + if M3uCache.Revalidating { + M3uCache.RUnlock() + + for { + <-time.After(time.Second) + M3uCache.RLock() + if !M3uCache.Revalidating { + M3uCache.RUnlock() + return readCacheFromFile() + } + M3uCache.RUnlock() + } + } + M3uCache.RUnlock() + + debug := isDebugMode() + if debug { + utils.SafeLogln("[DEBUG] Regenerating M3U cache in the background") + } + + baseURL := utils.DetermineBaseURL(r) + if debug { + utils.SafeLogf("[DEBUG] Base URL set to %s\n", baseURL) + } + + var content strings.Builder + content.WriteString("#EXTM3U\n") + + streams := GetStreams() + for _, stream := range streams { + if len(stream.URLs) == 0 { + continue + } + + if debug { + utils.SafeLogf("[DEBUG] Processing stream with TVG ID: %s\n", stream.TvgID) + } + + content.WriteString(formatStreamEntry(baseURL, stream)) + } + + if debug { + utils.SafeLogln("[DEBUG] Finished generating M3U content") + } + + stringContent := content.String() + + if err := writeCacheToFile(stringContent); err != nil { + utils.SafeLogf("Error writing cache to file: %v\n", err) + } + + M3uCache.Lock() + M3uCache.Exists = true + M3uCache.Revalidating = false + M3uCache.Unlock() + + return stringContent +} + +func ClearCache() { + debug := isDebugMode() + + M3uCache.Lock() + defer M3uCache.Unlock() + + M3uCache.Exists = false + + if debug { + utils.SafeLogln("[DEBUG] Clearing memory and disk M3U cache.") + } + if err := deleteCacheFile(); err != nil && debug { + utils.SafeLogf("[DEBUG] Cache file deletion failed: %v\n", err) + } +} + +func readCacheFromFile() string { + debug := isDebugMode() + + data, err := os.ReadFile(cacheFilePath) + if err != nil { + if debug { + utils.SafeLogf("[DEBUG] Cache file reading failed: %v\n", err) + } + + return "#EXTM3U\n" + } + return string(data) +} + +func writeCacheToFile(content string) error { + return os.WriteFile(cacheFilePath, []byte(content), 0644) +} + +func deleteCacheFile() error { + return os.Remove(cacheFilePath) +} + +func formatStreamEntry(baseURL string, stream StreamInfo) string { + var entry strings.Builder + + extInfTags := []string{"#EXTINF:-1"} + if stream.TvgID != "" { + extInfTags = append(extInfTags, fmt.Sprintf("tvg-id=\"%s\"", stream.TvgID)) + } + if stream.TvgChNo != "" { + extInfTags = append(extInfTags, fmt.Sprintf("tvg-chno=\"%s\"", stream.TvgChNo)) + } + if stream.LogoURL != "" { + extInfTags = append(extInfTags, fmt.Sprintf("tvg-logo=\"%s\"", stream.LogoURL)) + } + extInfTags = append(extInfTags, fmt.Sprintf("tvg-name=\"%s\"", stream.Title)) + extInfTags = append(extInfTags, fmt.Sprintf("group-title=\"%s\"", stream.Group)) + + entry.WriteString(fmt.Sprintf("%s,%s\n", strings.Join(extInfTags, " "), stream.Title)) + entry.WriteString(GenerateStreamURL(baseURL, stream)) + entry.WriteString("\n") + + return entry.String() +} diff --git a/store/concurrency.go b/store/concurrency.go new file mode 100644 index 00000000..e4a3f03d --- /dev/null +++ b/store/concurrency.go @@ -0,0 +1,72 @@ +package store + +import ( + "fmt" + "m3u-stream-merger/utils" + "os" + "strconv" + "sync" +) + +type ConcurrencyManager struct { + mu sync.Mutex + count map[int]int +} + +func NewConcurrencyManager() *ConcurrencyManager { + return &ConcurrencyManager{count: make(map[int]int)} +} + +func (cm *ConcurrencyManager) Increment(m3uIndex int) { + cm.mu.Lock() + defer cm.mu.Unlock() + cm.count[m3uIndex]++ +} + +func (cm *ConcurrencyManager) Decrement(m3uIndex int) { + cm.mu.Lock() + defer cm.mu.Unlock() + if cm.count[m3uIndex] > 0 { + cm.count[m3uIndex]-- + } +} + +func (cm *ConcurrencyManager) GetCount(m3uIndex int) int { + cm.mu.Lock() + defer cm.mu.Unlock() + return cm.count[m3uIndex] +} + +func (cm *ConcurrencyManager) ConcurrencyPriorityValue(m3uIndex int) int { + maxConcurrency, err := strconv.Atoi(os.Getenv(fmt.Sprintf("M3U_MAX_CONCURRENCY_%d", m3uIndex+1))) + if err != nil { + maxConcurrency = 1 + } + + count := cm.GetCount(m3uIndex) + + return maxConcurrency - count +} + +func (cm *ConcurrencyManager) CheckConcurrency(m3uIndex int) bool { + maxConcurrency, err := strconv.Atoi(os.Getenv(fmt.Sprintf("M3U_MAX_CONCURRENCY_%d", m3uIndex+1))) + if err != nil { + maxConcurrency = 1 + } + + count := cm.GetCount(m3uIndex) + + utils.SafeLogf("Current number of connections for M3U_%d: %d", m3uIndex+1, count) + return count >= maxConcurrency +} + +func (cm *ConcurrencyManager) UpdateConcurrency(m3uIndex int, incr bool) { + if incr { + cm.Increment(m3uIndex) + } else { + cm.Decrement(m3uIndex) + } + + count := cm.GetCount(m3uIndex) + utils.SafeLogf("Current number of connections for M3U_%d: %d", m3uIndex+1, count) +} diff --git a/store/downloader.go b/store/downloader.go new file mode 100644 index 00000000..c6d77bcc --- /dev/null +++ b/store/downloader.go @@ -0,0 +1,90 @@ +package store + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "m3u-stream-merger/utils" +) + +func DownloadM3USource(m3uIndex int) (err error) { + debug := os.Getenv("DEBUG") == "true" + m3uURL := os.Getenv(fmt.Sprintf("M3U_URL_%d", m3uIndex+1)) + + if debug { + utils.SafeLogf("[DEBUG] Processing M3U from: %s\n", m3uURL) + } + + finalPath := utils.GetM3UFilePathByIndex(m3uIndex + 1) + tmpPath := finalPath + ".new" + + // Handle local file URLs + if strings.HasPrefix(m3uURL, "file://") { + localPath := strings.TrimPrefix(m3uURL, "file://") + if debug { + utils.SafeLogf("[DEBUG] Local M3U file detected: %s\n", localPath) + } + + // Ensure finalPath's directory exists + err := os.MkdirAll(filepath.Dir(finalPath), os.ModePerm) + if err != nil { + return fmt.Errorf("Error creating directories for final path: %v", err) + } + + // Create a symlink + err = os.Symlink(localPath, finalPath) + if err != nil { + return fmt.Errorf("Error creating symlink: %v", err) + } + + if debug { + utils.SafeLogf("[DEBUG] Symlink created from %s to %s\n", localPath, finalPath) + } + + return nil + } + + // Handle remote URLs + if debug { + utils.SafeLogf("[DEBUG] Remote M3U URL detected: %s\n", m3uURL) + } + + resp, err := utils.CustomHttpRequest("GET", m3uURL) + if err != nil { + return fmt.Errorf("HTTP GET error: %v", err) + } + defer func() { + _, _ = io.Copy(io.Discard, resp.Body) // Discard remaining body content + resp.Body.Close() + }() + + // Ensure finalPath's directory exists + err = os.MkdirAll(filepath.Dir(finalPath), os.ModePerm) + if err != nil { + return fmt.Errorf("Error creating directories for final path: %v", err) + } + + // Write response body to finalPath + outFile, err := os.Create(tmpPath) + if err != nil { + return fmt.Errorf("Error creating file: %v", err) + } + defer outFile.Close() + + _, err = io.Copy(outFile, resp.Body) + if err != nil { + return fmt.Errorf("Error writing to file: %v", err) + } + + _ = os.Remove(finalPath) + _ = os.Rename(tmpPath, finalPath) + + if debug { + utils.SafeLogf("[DEBUG] M3U file downloaded to %s\n", finalPath) + } + + return nil +} diff --git a/m3u/filters.go b/store/filters.go similarity index 92% rename from m3u/filters.go rename to store/filters.go index a491b2f7..e87f9812 100644 --- a/m3u/filters.go +++ b/store/filters.go @@ -1,7 +1,6 @@ -package m3u +package store import ( - "m3u-stream-merger/database" "m3u-stream-merger/utils" "regexp" ) @@ -10,7 +9,7 @@ var includeFilters [][]string var excludeFilters [][]string var filtersInitialized bool -func checkFilter(stream database.StreamInfo) bool { +func checkFilter(stream StreamInfo) bool { if !filtersInitialized { excludeFilters = [][]string{ utils.GetFilters("EXCLUDE_GROUPS"), diff --git a/store/parser.go b/store/parser.go new file mode 100644 index 00000000..3574a498 --- /dev/null +++ b/store/parser.go @@ -0,0 +1,122 @@ +package store + +import ( + "bufio" + "os" + "regexp" + "strings" + + "m3u-stream-merger/utils" + + "github.com/gosimple/slug" +) + +func ParseStreamInfoBySlug(slug string) (*StreamInfo, error) { + return DecodeSlug(slug) +} + +func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo, lineNumber int)) error { + utils.SafeLogln("Parsing M3U file.") + filePath := utils.GetM3UFilePathByIndex(m3uIndex) + + file, err := os.Open(filePath) + if err != nil { + return err + } + + scanner := bufio.NewScanner(file) + + currentLine := 0 + for scanner.Scan() { + currentLine++ + line := scanner.Text() + if strings.HasPrefix(line, "#EXTINF:") { + if scanner.Scan() { + nextLine := scanner.Text() + // skip all other #EXT tags + for strings.HasPrefix(nextLine, "#") { + if scanner.Scan() { + nextLine = scanner.Text() + } else { + break + } + } + + streamInfo := parseLine(line, nextLine, m3uIndex) + + if !checkFilter(streamInfo) { + continue + } + + fn(streamInfo, currentLine) + } + } + } + + return nil +} + +func parseLine(line string, nextLine string, m3uIndex int) StreamInfo { + debug := os.Getenv("DEBUG") == "true" + if debug { + utils.SafeLogf("[DEBUG] Parsing line: %s\n", line) + utils.SafeLogf("[DEBUG] Next line: %s\n", nextLine) + utils.SafeLogf("[DEBUG] M3U index: %d\n", m3uIndex) + } + + currentStream := StreamInfo{} + currentStream.URLs = map[int]string{m3uIndex: strings.TrimSpace(nextLine)} + + lineWithoutPairs := line + + // Define a regular expression to capture key-value pairs + regex := regexp.MustCompile(`([a-zA-Z0-9_-]+)="([^"]+)"`) + + // Find all key-value pairs in the line + matches := regex.FindAllStringSubmatch(line, -1) + + for _, match := range matches { + key := strings.TrimSpace(match[1]) + value := strings.TrimSpace(match[2]) + + if debug { + utils.SafeLogf("[DEBUG] Processing attribute: %s=%s\n", key, value) + } + + switch strings.ToLower(key) { + case "tvg-id": + currentStream.TvgID = utils.TvgIdParser(value) + case "tvg-chno": + currentStream.TvgChNo = utils.TvgChNoParser(value) + case "tvg-name": + currentStream.Title = utils.TvgNameParser(value) + case "group-title": + currentStream.Group = utils.GroupTitleParser(value) + case "tvg-logo": + currentStream.LogoURL = utils.TvgLogoParser(value) + default: + if debug { + utils.SafeLogf("[DEBUG] Uncaught attribute: %s=%s\n", key, value) + } + } + + lineWithoutPairs = strings.Replace(lineWithoutPairs, match[0], "", 1) + } + + lineCommaSplit := strings.SplitN(lineWithoutPairs, ",", 2) + + if len(lineCommaSplit) > 1 { + if debug { + utils.SafeLogf("[DEBUG] Line comma split detected, title: %s\n", strings.TrimSpace(lineCommaSplit[1])) + } + currentStream.Title = utils.TvgNameParser(strings.TrimSpace(lineCommaSplit[1])) + } + + currentStream.Slug = slug.Make(currentStream.Title) + + if debug { + utils.SafeLogf("[DEBUG] Generated slug: %s\n", currentStream.Slug) + } + + return currentStream +} diff --git a/store/slug.go b/store/slug.go new file mode 100644 index 00000000..7195395e --- /dev/null +++ b/store/slug.go @@ -0,0 +1,58 @@ +package store + +import ( + "bytes" + "encoding/base32" + "fmt" + "io" + + "github.com/goccy/go-json" + "github.com/klauspost/compress/zstd" +) + +func EncodeSlug(stream StreamInfo) string { + jsonData, err := json.Marshal(stream) + if err != nil { + return "" + } + + var compressedData bytes.Buffer + writer, err := zstd.NewWriter(&compressedData) + if err != nil { + return "" + } + + _, err = writer.Write(jsonData) + if err != nil { + return "" + } + writer.Close() + + encodedData := base32.StdEncoding.EncodeToString(compressedData.Bytes()) + return encodedData +} + +func DecodeSlug(encodedSlug string) (*StreamInfo, error) { + decodedData, err := base32.StdEncoding.DecodeString(encodedSlug) + if err != nil { + return nil, fmt.Errorf("error decoding Base32 data: %v", err) + } + + reader, err := zstd.NewReader(bytes.NewReader(decodedData)) + if err != nil { + return nil, fmt.Errorf("error creating zstd reader: %v", err) + } + decompressedData, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("error reading decompressed data: %v", err) + } + reader.Close() + + var result StreamInfo + err = json.Unmarshal(decompressedData, &result) + if err != nil { + return nil, fmt.Errorf("error deserializing data: %v", err) + } + + return &result, nil +} diff --git a/store/streams.go b/store/streams.go new file mode 100644 index 00000000..433642fa --- /dev/null +++ b/store/streams.go @@ -0,0 +1,116 @@ +package store + +import ( + "fmt" + "m3u-stream-merger/utils" + "math" + "os" + "sort" + "strconv" + "strings" +) + +func GetStreamBySlug(slug string) (StreamInfo, error) { + streamInfo, err := ParseStreamInfoBySlug(slug) + if err != nil { + return StreamInfo{}, fmt.Errorf("error parsing stream info: %v", err) + } + + return *streamInfo, nil +} + +func GetStreams() []StreamInfo { + var ( + debug = os.Getenv("DEBUG") == "true" + streams = make(map[string]StreamInfo) // Map to store unique streams by title + result = make([]StreamInfo, 0) // Slice to store final results + ) + + for _, m3uIndex := range utils.GetM3UIndexes() { + err := M3UScanner(m3uIndex, func(streamInfo StreamInfo, lineNumber int) { + // Check uniqueness and update if necessary + if existingStream, exists := streams[streamInfo.Title]; exists { + for idx, url := range streamInfo.URLs { + existingStream.URLs[idx] = url + } + streams[streamInfo.Title] = existingStream + } else { + streams[streamInfo.Title] = streamInfo + } + }) + if err != nil && debug { + utils.SafeLogf("error getting streams: %v\n", err) + } + } + + for _, stream := range streams { + result = append(result, stream) + } + + sort.Slice(result, func(i, j int) bool { + return calculateSortScore(result[i]) < calculateSortScore(result[j]) + }) + + return result +} + +func GenerateStreamURL(baseUrl string, stream StreamInfo) string { + var subPath string + var err error + for _, srcUrl := range stream.URLs { + subPath, err = utils.GetSubPathFromUrl(srcUrl) + if err != nil { + continue + } + + ext, err := utils.GetFileExtensionFromUrl(srcUrl) + if err != nil { + return fmt.Sprintf("%s/proxy/%s/%s\n", baseUrl, subPath, stream.Slug) + } + + return fmt.Sprintf("%s/proxy/%s/%s%s\n", baseUrl, subPath, stream.Slug, ext) + } + return fmt.Sprintf("%s/proxy/stream/%s\n", baseUrl, stream.Slug) +} + +func getSortingValue(s StreamInfo) string { + key := os.Getenv("SORTING_KEY") + + var value string + switch key { + case "tvg-id": + value = s.TvgID + case "tvg-chno": + value = s.TvgChNo + default: + value = s.TvgID + } + + // Try to parse the value as a float. + if numValue, err := strconv.ParseFloat(value, 64); err == nil { + return fmt.Sprintf("%010.2f", numValue) + s.Title + } + + // If parsing fails, fall back to using the original string value. + return value + s.Title +} + +func calculateSortScore(s StreamInfo) float64 { + // Add to the sorted set with tvg_id as the score + maxLen := 40 + base := float64(256) + + // Normalize length by padding the string + paddedString := strings.ToLower(getSortingValue(s)) + if len(paddedString) < maxLen { + paddedString = paddedString + strings.Repeat("\x00", maxLen-len(paddedString)) + } + + sortScore := 0.0 + for i := 0; i < len(paddedString); i++ { + charValue := float64(paddedString[i]) + sortScore += charValue / math.Pow(base, float64(i+1)) + } + + return sortScore +} diff --git a/database/types.go b/store/types.go similarity index 91% rename from database/types.go rename to store/types.go index 48fe399e..2dff4c27 100644 --- a/database/types.go +++ b/store/types.go @@ -1,4 +1,4 @@ -package database +package store type StreamInfo struct { Slug string `json:"slug"` diff --git a/proxy/proxy_test.go b/tests/proxy_test.go similarity index 62% rename from proxy/proxy_test.go rename to tests/proxy_test.go index eccaf72d..2a769369 100644 --- a/proxy/proxy_test.go +++ b/tests/proxy_test.go @@ -1,13 +1,11 @@ -package proxy +package tests import ( "bytes" - "context" "io" "log" - "m3u-stream-merger/database" - "m3u-stream-merger/m3u" - "m3u-stream-merger/updater" + "m3u-stream-merger/handlers" + "m3u-stream-merger/store" "net/http" "net/http/httptest" "os" @@ -16,35 +14,17 @@ import ( ) func TestStreamHandler(t *testing.T) { - os.Setenv("REDIS_ADDR", "127.0.0.1:6379") - os.Setenv("REDIS_PASS", "") - os.Setenv("REDIS_DB", "0") - - db, err := database.InitializeDb() - if err != nil { - t.Errorf("InitializeDb returned error: %v", err) - } - - err = db.ClearDb() - if err != nil { - t.Errorf("ClearDb returned error: %v", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - os.Setenv("M3U_URL_1", "https://gist.githubusercontent.com/sonroyaalmerol/de1c90e8681af040924da5d15c7f530d/raw/06844df09e69ea278060252ca5aa8d767eb4543d/test-m3u.m3u") os.Setenv("INCLUDE_GROUPS_1", "movies") - _, err = updater.Initialize(ctx) + err := store.DownloadM3USource(1) if err != nil { - t.Errorf("Updater returned error: %v", err) + t.Errorf("Downloader returned error: %v", err) } - streamChan := db.GetStreams() - streams := []database.StreamInfo{} + streams := store.GetStreams() - for stream := range streamChan { + for _, stream := range streams { streams = append(streams, stream) } @@ -52,7 +32,7 @@ func TestStreamHandler(t *testing.T) { m3uW := httptest.NewRecorder() func() { - m3u.Handler(m3uW, m3uReq) + handlers.M3UHandler(m3uW, m3uReq) }() m3uResp := m3uW.Result() @@ -60,15 +40,17 @@ func TestStreamHandler(t *testing.T) { t.Errorf("Playlist Route - Expected status code %d, got %d", http.StatusOK, m3uResp.StatusCode) } + cm := store.NewConcurrencyManager() + for _, stream := range streams { log.Printf("Stream (%s): %v", stream.Title, stream) - genStreamUrl := strings.TrimSpace(m3u.GenerateStreamURL("", stream)) + genStreamUrl := strings.TrimSpace(store.GenerateStreamURL("", stream)) req := httptest.NewRequest("GET", genStreamUrl, nil) w := httptest.NewRecorder() // Call the handler function - Handler(w, req) + handlers.StreamHandler(w, req, cm) // Check the response status code resp := w.Result() diff --git a/updater/updater.go b/updater/updater.go index 577a5417..6e6d9eb8 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -2,9 +2,7 @@ package updater import ( "context" - "fmt" - "m3u-stream-merger/database" - "m3u-stream-merger/m3u" + "m3u-stream-merger/store" "m3u-stream-merger/utils" "os" "strings" @@ -15,30 +13,19 @@ import ( type Updater struct { sync.Mutex - ctx context.Context - db *database.Instance - Cron *cron.Cron - M3UParser *m3u.Parser + ctx context.Context + Cron *cron.Cron } func Initialize(ctx context.Context) (*Updater, error) { - db, err := database.InitializeDb() - if err != nil { - utils.SafeLogf("Error initializing Redis database: %v", err) - return nil, err - } - clearOnBoot := os.Getenv("CLEAR_ON_BOOT") if len(strings.TrimSpace(clearOnBoot)) == 0 { clearOnBoot = "false" } if clearOnBoot == "true" { - utils.SafeLogln("CLEAR_ON_BOOT enabled. Clearing current database.") - if err := db.ClearDb(); err != nil { - utils.SafeLogf("Error clearing database: %v", err) - return nil, err - } + utils.SafeLogln("CLEAR_ON_BOOT enabled. Clearing current cache.") + store.ClearCache() } cronSched := os.Getenv("SYNC_CRON") @@ -48,13 +35,11 @@ func Initialize(ctx context.Context) (*Updater, error) { } updateInstance := &Updater{ - ctx: ctx, - db: db, - M3UParser: m3u.InitializeParser(), + ctx: ctx, } c := cron.New() - _, err = c.AddFunc(cronSched, func() { + _, err := c.AddFunc(cronSched, func() { go updateInstance.UpdateSources(ctx) }) if err != nil { @@ -79,79 +64,44 @@ func Initialize(ctx context.Context) (*Updater, error) { return updateInstance, nil } -func (instance *Updater) UpdateSource(m3uUrl string, index int) { - utils.SafeLogf("Background process: Updating M3U #%d from %s\n", index+1, m3uUrl) - err := instance.M3UParser.ParseURL(m3uUrl, index) - if err != nil { - utils.SafeLogf("Background process: Error updating M3U: %v\n", err) - } else { - utils.SafeLogf("Background process: Updated M3U #%d from %s\n", index+1, m3uUrl) - } -} - func (instance *Updater) UpdateSources(ctx context.Context) { // Ensure only one job is running at a time instance.Lock() defer instance.Unlock() - db, err := database.InitializeDb() - if err != nil { - utils.SafeLogln("Background process: Failed to initialize db connection.") - return - } - select { case <-ctx.Done(): return default: utils.SafeLogln("Background process: Checking M3U_URLs...") var wg sync.WaitGroup - index := 0 - for { - m3uUrl, m3uExists := os.LookupEnv(fmt.Sprintf("M3U_URL_%d", index+1)) - if !m3uExists { - break - } - - utils.SafeLogf("Background process: Fetching M3U_URL_%d...\n", index+1) + indexes := utils.GetM3UIndexes() + for _, idx := range indexes { + utils.SafeLogf("Background process: Fetching M3U_URL_%d...\n", idx+1) wg.Add(1) // Start the goroutine for periodic updates - go func(m3uUrl string, index int) { + go func(idx int) { defer wg.Done() - instance.UpdateSource(m3uUrl, index) - }(m3uUrl, index) - - index++ + store.DownloadM3USource(idx) + }(idx) } wg.Wait() - utils.SafeLogf("Background process: M3U fetching complete. Saving to database...\n") - - err := db.SaveToDb(instance.M3UParser.GetStreams()) - if err != nil { - utils.SafeLogf("Background process: Error updating M3U database: %v\n", err) - utils.SafeLogln("Background process: Clearing database after error in attempt to fix issue after container restart.") - - if err := db.ClearDb(); err != nil { - utils.SafeLogf("Background process: Error clearing database: %v\n", err) - } - } - - m3u.ClearCache() + utils.SafeLogf("Background process: M3U fetching complete.\n") cacheOnSync := os.Getenv("CACHE_ON_SYNC") if len(strings.TrimSpace(cacheOnSync)) == 0 { cacheOnSync = "false" } - utils.SafeLogln("Background process: Updated M3U database.") + utils.SafeLogln("Background process: Updated M3U store.") if cacheOnSync == "true" { if _, ok := os.LookupEnv("BASE_URL"); !ok { utils.SafeLogln("BASE_URL is required for CACHE_ON_SYNC to work.") } utils.SafeLogln("CACHE_ON_SYNC enabled. Building cache.") - m3u.InitCache(db) + _ = store.RevalidatingGetM3U(nil) } } } diff --git a/utils/checksum.go b/utils/checksum.go new file mode 100644 index 00000000..4bc0ff8a --- /dev/null +++ b/utils/checksum.go @@ -0,0 +1,11 @@ +package utils + +import ( + "crypto/sha256" + "encoding/hex" +) + +func CalculateChecksum(data string) string { + hash := sha256.Sum256([]byte(data)) + return hex.EncodeToString(hash[:]) +} diff --git a/utils/m3u_path.go b/utils/m3u_path.go new file mode 100644 index 00000000..e40fd0da --- /dev/null +++ b/utils/m3u_path.go @@ -0,0 +1,19 @@ +package utils + +import "fmt" + +func GetM3UFilePathByIndex(m3uIndex int) string { + m3uFile := fmt.Sprintf("/m3u-proxy/sources/%d.m3u", m3uIndex) + + return m3uFile +} + +func GetAllM3UFilePaths() []string { + paths := []string{} + m3uIndexes := GetM3UIndexes() + for _, idx := range m3uIndexes { + paths = append(paths, GetM3UFilePathByIndex(idx)) + } + + return paths +} diff --git a/utils/middlewares.go b/utils/middlewares.go new file mode 100644 index 00000000..6dbe6e6a --- /dev/null +++ b/utils/middlewares.go @@ -0,0 +1,46 @@ +package utils + +import ( + "os" + "regexp" + "strings" +) + +func GeneralParser(value string) string { + if strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) { + value = strings.Trim(value, `"`) + } + + return value +} + +func TvgNameParser(value string) string { + substrFilter := os.Getenv("TITLE_SUBSTR_FILTER") + // Apply character filter + if substrFilter != "" { + re, err := regexp.Compile(substrFilter) + if err != nil { + SafeLogf("Error compiling character filter regex: %v\n", err) + } else { + value = re.ReplaceAllString(value, "") + } + } + + return GeneralParser(value) +} + +func TvgIdParser(value string) string { + return GeneralParser(value) +} + +func TvgChNoParser(value string) string { + return GeneralParser(value) +} + +func GroupTitleParser(value string) string { + return GeneralParser(value) +} + +func TvgLogoParser(value string) string { + return GeneralParser(value) +} diff --git a/utils/stream_url.go b/utils/stream_url.go new file mode 100644 index 00000000..829ddd0b --- /dev/null +++ b/utils/stream_url.go @@ -0,0 +1,31 @@ +package utils + +import ( + "net/url" + "path" + "strings" +) + +func GetFileExtensionFromUrl(rawUrl string) (string, error) { + u, err := url.Parse(rawUrl) + if err != nil { + return "", err + } + return path.Ext(u.Path), nil +} + +func GetSubPathFromUrl(rawUrl string) (string, error) { + parsedURL, err := url.Parse(rawUrl) + if err != nil { + return "", err + } + + pathSegments := strings.Split(parsedURL.Path, "/") + + if len(pathSegments) <= 1 { + return "stream", nil + } + + basePath := strings.Join(pathSegments[1:len(pathSegments)-1], "/") + return basePath, nil +} From b8eb391ae93336c71176c2b9262f360e92c111d7 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 20:00:57 -0500 Subject: [PATCH 02/49] fix test source index + lint errors --- tests/proxy_test.go | 6 +----- updater/updater.go | 7 ++++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/proxy_test.go b/tests/proxy_test.go index 2a769369..7adbda77 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -17,17 +17,13 @@ func TestStreamHandler(t *testing.T) { os.Setenv("M3U_URL_1", "https://gist.githubusercontent.com/sonroyaalmerol/de1c90e8681af040924da5d15c7f530d/raw/06844df09e69ea278060252ca5aa8d767eb4543d/test-m3u.m3u") os.Setenv("INCLUDE_GROUPS_1", "movies") - err := store.DownloadM3USource(1) + err := store.DownloadM3USource(0) if err != nil { t.Errorf("Downloader returned error: %v", err) } streams := store.GetStreams() - for _, stream := range streams { - streams = append(streams, stream) - } - m3uReq := httptest.NewRequest("GET", "/playlist.m3u", nil) m3uW := httptest.NewRecorder() diff --git a/updater/updater.go b/updater/updater.go index 6e6d9eb8..d060d97c 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -65,6 +65,8 @@ func Initialize(ctx context.Context) (*Updater, error) { } func (instance *Updater) UpdateSources(ctx context.Context) { + debug := os.Getenv("DEBUG") == "true" + // Ensure only one job is running at a time instance.Lock() defer instance.Unlock() @@ -83,7 +85,10 @@ func (instance *Updater) UpdateSources(ctx context.Context) { // Start the goroutine for periodic updates go func(idx int) { defer wg.Done() - store.DownloadM3USource(idx) + err := store.DownloadM3USource(idx) + if err != nil && debug { + utils.SafeLogf("Background process: Error fetching M3U_URL_%d: %v\n", idx+1, err) + } }(idx) } wg.Wait() From 8061320e812b367472d1f4cfaa230367bd57aaed Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 20:10:01 -0500 Subject: [PATCH 03/49] remove unnecessary remnants --- store/downloader.go | 2 ++ store/parser.go | 8 +++----- store/streams.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/store/downloader.go b/store/downloader.go index c6d77bcc..9dce7262 100644 --- a/store/downloader.go +++ b/store/downloader.go @@ -34,6 +34,8 @@ func DownloadM3USource(m3uIndex int) (err error) { return fmt.Errorf("Error creating directories for final path: %v", err) } + _ = os.Remove(finalPath) + // Create a symlink err = os.Symlink(localPath, finalPath) if err != nil { diff --git a/store/parser.go b/store/parser.go index 3574a498..7cb292b7 100644 --- a/store/parser.go +++ b/store/parser.go @@ -15,8 +15,8 @@ func ParseStreamInfoBySlug(slug string) (*StreamInfo, error) { return DecodeSlug(slug) } -func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo, lineNumber int)) error { - utils.SafeLogln("Parsing M3U file.") +func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo)) error { + utils.SafeLogf("Parsing M3U %d...\n", m3uIndex) filePath := utils.GetM3UFilePathByIndex(m3uIndex) file, err := os.Open(filePath) @@ -26,9 +26,7 @@ func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo, lineNumber int)) er scanner := bufio.NewScanner(file) - currentLine := 0 for scanner.Scan() { - currentLine++ line := scanner.Text() if strings.HasPrefix(line, "#EXTINF:") { if scanner.Scan() { @@ -48,7 +46,7 @@ func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo, lineNumber int)) er continue } - fn(streamInfo, currentLine) + fn(streamInfo) } } } diff --git a/store/streams.go b/store/streams.go index 433642fa..fb7dc2e4 100644 --- a/store/streams.go +++ b/store/streams.go @@ -27,7 +27,7 @@ func GetStreams() []StreamInfo { ) for _, m3uIndex := range utils.GetM3UIndexes() { - err := M3UScanner(m3uIndex, func(streamInfo StreamInfo, lineNumber int) { + err := M3UScanner(m3uIndex, func(streamInfo StreamInfo) { // Check uniqueness and update if necessary if existingStream, exists := streams[streamInfo.Title]; exists { for idx, url := range streamInfo.URLs { From 2623f0e59ff8f64073c8473a0f73394d11a59f9b Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 20:31:06 -0500 Subject: [PATCH 04/49] use channel to avoid deadlock --- store/cache.go | 34 ++++++++++++++-------------------- store/downloader.go | 2 +- store/parser.go | 2 +- utils/m3u_path.go | 2 +- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/store/cache.go b/store/cache.go index c257846f..7e48e7a5 100644 --- a/store/cache.go +++ b/store/cache.go @@ -7,16 +7,18 @@ import ( "os" "strings" "sync" - "time" ) type Cache struct { sync.RWMutex - Exists bool - Revalidating bool + Exists bool + Revalidating bool + generationDone chan struct{} } -var M3uCache = &Cache{} +var M3uCache = &Cache{ + generationDone: make(chan struct{}), +} const cacheFilePath = "/m3u-proxy/cache.m3u" @@ -36,6 +38,10 @@ func RevalidatingGetM3U(r *http.Request) string { } M3uCache.RUnlock() + + // Wait for generation to finish + <-M3uCache.generationDone + return readCacheFromFile() } M3uCache.RUnlock() @@ -44,22 +50,6 @@ func RevalidatingGetM3U(r *http.Request) string { } func generateM3UContent(r *http.Request) string { - M3uCache.RLock() - if M3uCache.Revalidating { - M3uCache.RUnlock() - - for { - <-time.After(time.Second) - M3uCache.RLock() - if !M3uCache.Revalidating { - M3uCache.RUnlock() - return readCacheFromFile() - } - M3uCache.RUnlock() - } - } - M3uCache.RUnlock() - debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Regenerating M3U cache in the background") @@ -101,6 +91,9 @@ func generateM3UContent(r *http.Request) string { M3uCache.Revalidating = false M3uCache.Unlock() + close(M3uCache.generationDone) + M3uCache.generationDone = make(chan struct{}) + return stringContent } @@ -111,6 +104,7 @@ func ClearCache() { defer M3uCache.Unlock() M3uCache.Exists = false + M3uCache.Revalidating = false if debug { utils.SafeLogln("[DEBUG] Clearing memory and disk M3U cache.") diff --git a/store/downloader.go b/store/downloader.go index 9dce7262..5888e0ed 100644 --- a/store/downloader.go +++ b/store/downloader.go @@ -18,7 +18,7 @@ func DownloadM3USource(m3uIndex int) (err error) { utils.SafeLogf("[DEBUG] Processing M3U from: %s\n", m3uURL) } - finalPath := utils.GetM3UFilePathByIndex(m3uIndex + 1) + finalPath := utils.GetM3UFilePathByIndex(m3uIndex) tmpPath := finalPath + ".new" // Handle local file URLs diff --git a/store/parser.go b/store/parser.go index 7cb292b7..bcccc79c 100644 --- a/store/parser.go +++ b/store/parser.go @@ -16,7 +16,7 @@ func ParseStreamInfoBySlug(slug string) (*StreamInfo, error) { } func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo)) error { - utils.SafeLogf("Parsing M3U %d...\n", m3uIndex) + utils.SafeLogf("Parsing M3U #%d...\n", m3uIndex+1) filePath := utils.GetM3UFilePathByIndex(m3uIndex) file, err := os.Open(filePath) diff --git a/utils/m3u_path.go b/utils/m3u_path.go index e40fd0da..59985989 100644 --- a/utils/m3u_path.go +++ b/utils/m3u_path.go @@ -3,7 +3,7 @@ package utils import "fmt" func GetM3UFilePathByIndex(m3uIndex int) string { - m3uFile := fmt.Sprintf("/m3u-proxy/sources/%d.m3u", m3uIndex) + m3uFile := fmt.Sprintf("/m3u-proxy/sources/%d.m3u", m3uIndex+1) return m3uFile } From 73a8e6bb995c871c6846c2bd94a8acd580da0eef Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 20:43:36 -0500 Subject: [PATCH 05/49] fix m3u generate and add more revalidating logs --- store/cache.go | 20 ++++++++++++++++++++ store/parser.go | 4 +--- store/streams.go | 6 +++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/store/cache.go b/store/cache.go index 7e48e7a5..80be16ed 100644 --- a/store/cache.go +++ b/store/cache.go @@ -27,9 +27,19 @@ func isDebugMode() bool { } func RevalidatingGetM3U(r *http.Request) string { + debug := isDebugMode() + if debug { + utils.SafeLogln("[DEBUG] Revalidating M3U cache") + } M3uCache.RLock() if M3uCache.Exists { + if debug { + utils.SafeLogln("[DEBUG] M3U cache already exists") + } if !M3uCache.Revalidating { + if debug { + utils.SafeLogln("[DEBUG] Initiating cache revalidation") + } M3uCache.Lock() M3uCache.Revalidating = true M3uCache.Unlock() @@ -38,14 +48,24 @@ func RevalidatingGetM3U(r *http.Request) string { } M3uCache.RUnlock() + if debug { + utils.SafeLogln("[DEBUG] Waiting for cache revalidation to finish") + } // Wait for generation to finish <-M3uCache.generationDone + if debug { + utils.SafeLogln("[DEBUG] Finished cache revalidation") + } return readCacheFromFile() } M3uCache.RUnlock() + if debug { + utils.SafeLogln("[DEBUG] Existing cache not found, generating content") + } + return generateM3UContent(r) } diff --git a/store/parser.go b/store/parser.go index bcccc79c..2ed394d7 100644 --- a/store/parser.go +++ b/store/parser.go @@ -7,8 +7,6 @@ import ( "strings" "m3u-stream-merger/utils" - - "github.com/gosimple/slug" ) func ParseStreamInfoBySlug(slug string) (*StreamInfo, error) { @@ -110,7 +108,7 @@ func parseLine(line string, nextLine string, m3uIndex int) StreamInfo { currentStream.Title = utils.TvgNameParser(strings.TrimSpace(lineCommaSplit[1])) } - currentStream.Slug = slug.Make(currentStream.Title) + currentStream.Slug = EncodeSlug(currentStream) if debug { utils.SafeLogf("[DEBUG] Generated slug: %s\n", currentStream.Slug) diff --git a/store/streams.go b/store/streams.go index fb7dc2e4..d16a3c09 100644 --- a/store/streams.go +++ b/store/streams.go @@ -65,12 +65,12 @@ func GenerateStreamURL(baseUrl string, stream StreamInfo) string { ext, err := utils.GetFileExtensionFromUrl(srcUrl) if err != nil { - return fmt.Sprintf("%s/proxy/%s/%s\n", baseUrl, subPath, stream.Slug) + return fmt.Sprintf("%s/proxy/%s/%s", baseUrl, subPath, stream.Slug) } - return fmt.Sprintf("%s/proxy/%s/%s%s\n", baseUrl, subPath, stream.Slug, ext) + return fmt.Sprintf("%s/proxy/%s/%s%s", baseUrl, subPath, stream.Slug, ext) } - return fmt.Sprintf("%s/proxy/stream/%s\n", baseUrl, stream.Slug) + return fmt.Sprintf("%s/proxy/stream/%s", baseUrl, stream.Slug) } func getSortingValue(s StreamInfo) string { From e352228c74eacb3920fab60ae2422e00981ac7e8 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 20:49:53 -0500 Subject: [PATCH 06/49] deal with cache lock on generate --- store/cache.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/store/cache.go b/store/cache.go index 80be16ed..5165accd 100644 --- a/store/cache.go +++ b/store/cache.go @@ -40,9 +40,6 @@ func RevalidatingGetM3U(r *http.Request) string { if debug { utils.SafeLogln("[DEBUG] Initiating cache revalidation") } - M3uCache.Lock() - M3uCache.Revalidating = true - M3uCache.Unlock() go generateM3UContent(r) } @@ -70,6 +67,11 @@ func RevalidatingGetM3U(r *http.Request) string { } func generateM3UContent(r *http.Request) string { + M3uCache.Lock() + defer M3uCache.Unlock() + + M3uCache.Revalidating = true + debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Regenerating M3U cache in the background") @@ -106,10 +108,8 @@ func generateM3UContent(r *http.Request) string { utils.SafeLogf("Error writing cache to file: %v\n", err) } - M3uCache.Lock() M3uCache.Exists = true M3uCache.Revalidating = false - M3uCache.Unlock() close(M3uCache.generationDone) M3uCache.generationDone = make(chan struct{}) From 01c638acd3d2f4da9c601a9c4bb8931aedd6aaa5 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:02:16 -0500 Subject: [PATCH 07/49] use base64 and fix sorting --- store/slug.go | 32 ++++++++++++++++++++++++++----- store/streams.go | 50 +++++++++++------------------------------------- 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/store/slug.go b/store/slug.go index 7195395e..54078e2e 100644 --- a/store/slug.go +++ b/store/slug.go @@ -2,9 +2,10 @@ package store import ( "bytes" - "encoding/base32" + "encoding/base64" "fmt" "io" + "strings" "github.com/goccy/go-json" "github.com/klauspost/compress/zstd" @@ -28,17 +29,38 @@ func EncodeSlug(stream StreamInfo) string { } writer.Close() - encodedData := base32.StdEncoding.EncodeToString(compressedData.Bytes()) + encodedData := base64.StdEncoding.EncodeToString(compressedData.Bytes()) + + // 62nd char of encoding + encodedData = strings.Replace(encodedData, "+", "-", -1) + // 63rd char of encoding + encodedData = strings.Replace(encodedData, "/", "_", -1) + // Remove any trailing '='s + encodedData = strings.Replace(encodedData, "=", "", -1) + return encodedData } func DecodeSlug(encodedSlug string) (*StreamInfo, error) { - decodedData, err := base32.StdEncoding.DecodeString(encodedSlug) + decodedData, err := base64.StdEncoding.DecodeString(encodedSlug) if err != nil { - return nil, fmt.Errorf("error decoding Base32 data: %v", err) + return nil, fmt.Errorf("error decoding Base64 data: %v", err) + } + + strData := string(decodedData) + strData = strings.Replace(strData, "-", "+", -1) + strData = strings.Replace(strData, "_", "/", -1) + + switch len(strData) % 4 { + case 0: + case 1: + case 2: + strData += "==" + case 3: + strData += "=" } - reader, err := zstd.NewReader(bytes.NewReader(decodedData)) + reader, err := zstd.NewReader(bytes.NewReader([]byte(strData))) if err != nil { return nil, fmt.Errorf("error creating zstd reader: %v", err) } diff --git a/store/streams.go b/store/streams.go index d16a3c09..4b7f7979 100644 --- a/store/streams.go +++ b/store/streams.go @@ -3,11 +3,8 @@ package store import ( "fmt" "m3u-stream-merger/utils" - "math" "os" "sort" - "strconv" - "strings" ) func GetStreamBySlug(slug string) (StreamInfo, error) { @@ -47,9 +44,7 @@ func GetStreams() []StreamInfo { result = append(result, stream) } - sort.Slice(result, func(i, j int) bool { - return calculateSortScore(result[i]) < calculateSortScore(result[j]) - }) + sortStreams(result) return result } @@ -73,44 +68,21 @@ func GenerateStreamURL(baseUrl string, stream StreamInfo) string { return fmt.Sprintf("%s/proxy/stream/%s", baseUrl, stream.Slug) } -func getSortingValue(s StreamInfo) string { +func sortStreams(s []StreamInfo) { key := os.Getenv("SORTING_KEY") - var value string switch key { case "tvg-id": - value = s.TvgID + sort.Slice(s, func(i, j int) bool { + return s[i].TvgID < s[j].TvgID + }) case "tvg-chno": - value = s.TvgChNo + sort.Slice(s, func(i, j int) bool { + return s[i].TvgChNo < s[j].TvgChNo + }) default: - value = s.TvgID - } - - // Try to parse the value as a float. - if numValue, err := strconv.ParseFloat(value, 64); err == nil { - return fmt.Sprintf("%010.2f", numValue) + s.Title - } - - // If parsing fails, fall back to using the original string value. - return value + s.Title -} - -func calculateSortScore(s StreamInfo) float64 { - // Add to the sorted set with tvg_id as the score - maxLen := 40 - base := float64(256) - - // Normalize length by padding the string - paddedString := strings.ToLower(getSortingValue(s)) - if len(paddedString) < maxLen { - paddedString = paddedString + strings.Repeat("\x00", maxLen-len(paddedString)) - } - - sortScore := 0.0 - for i := 0; i < len(paddedString); i++ { - charValue := float64(paddedString[i]) - sortScore += charValue / math.Pow(base, float64(i+1)) + sort.Slice(s, func(i, j int) bool { + return s[i].Title < s[j].Title + }) } - - return sortScore } From 6f42b94431023396fddf567fa62541e9a56487f2 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:06:23 -0500 Subject: [PATCH 08/49] fix base64 decoding --- store/slug.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/store/slug.go b/store/slug.go index 54078e2e..6f372929 100644 --- a/store/slug.go +++ b/store/slug.go @@ -42,25 +42,22 @@ func EncodeSlug(stream StreamInfo) string { } func DecodeSlug(encodedSlug string) (*StreamInfo, error) { - decodedData, err := base64.StdEncoding.DecodeString(encodedSlug) - if err != nil { - return nil, fmt.Errorf("error decoding Base64 data: %v", err) - } - - strData := string(decodedData) - strData = strings.Replace(strData, "-", "+", -1) - strData = strings.Replace(strData, "_", "/", -1) + encodedSlug = strings.Replace(encodedSlug, "-", "+", -1) + encodedSlug = strings.Replace(encodedSlug, "_", "/", -1) - switch len(strData) % 4 { - case 0: - case 1: + switch len(encodedSlug) % 4 { case 2: - strData += "==" + encodedSlug += "==" case 3: - strData += "=" + encodedSlug += "=" + } + + decodedData, err := base64.StdEncoding.DecodeString(encodedSlug) + if err != nil { + return nil, fmt.Errorf("error decoding Base64 data: %v", err) } - reader, err := zstd.NewReader(bytes.NewReader([]byte(strData))) + reader, err := zstd.NewReader(bytes.NewReader(decodedData)) if err != nil { return nil, fmt.Errorf("error creating zstd reader: %v", err) } From ba295ca6525081d1dbe0193a8ddad898d5c0884e Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:12:21 -0500 Subject: [PATCH 09/49] use datadog zstd --- go.mod | 5 +---- go.sum | 8 ++------ store/slug.go | 24 ++++++------------------ 3 files changed, 9 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index d9771822..2e0dbbe9 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,7 @@ module m3u-stream-merger go 1.23.0 require ( + github.com/DataDog/zstd v1.5.6 github.com/goccy/go-json v0.10.3 - github.com/gosimple/slug v1.14.0 - github.com/klauspost/compress v1.17.11 github.com/robfig/cron/v3 v3.0.1 ) - -require github.com/gosimple/unidecode v1.0.1 // indirect diff --git a/go.sum b/go.sum index 13983400..8de3bbf3 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,6 @@ +github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= +github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/gosimple/slug v1.14.0 h1:RtTL/71mJNDfpUbCOmnf/XFkzKRtD6wL6Uy+3akm4Es= -github.com/gosimple/slug v1.14.0/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= -github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= -github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= diff --git a/store/slug.go b/store/slug.go index 6f372929..7e8448d2 100644 --- a/store/slug.go +++ b/store/slug.go @@ -1,14 +1,12 @@ package store import ( - "bytes" "encoding/base64" "fmt" - "io" "strings" + "github.com/DataDog/zstd" "github.com/goccy/go-json" - "github.com/klauspost/compress/zstd" ) func EncodeSlug(stream StreamInfo) string { @@ -17,19 +15,13 @@ func EncodeSlug(stream StreamInfo) string { return "" } - var compressedData bytes.Buffer - writer, err := zstd.NewWriter(&compressedData) + var compressedData []byte + _, err = zstd.CompressLevel(compressedData, jsonData, zstd.BestCompression) if err != nil { return "" } - _, err = writer.Write(jsonData) - if err != nil { - return "" - } - writer.Close() - - encodedData := base64.StdEncoding.EncodeToString(compressedData.Bytes()) + encodedData := base64.StdEncoding.EncodeToString(compressedData) // 62nd char of encoding encodedData = strings.Replace(encodedData, "+", "-", -1) @@ -57,15 +49,11 @@ func DecodeSlug(encodedSlug string) (*StreamInfo, error) { return nil, fmt.Errorf("error decoding Base64 data: %v", err) } - reader, err := zstd.NewReader(bytes.NewReader(decodedData)) - if err != nil { - return nil, fmt.Errorf("error creating zstd reader: %v", err) - } - decompressedData, err := io.ReadAll(reader) + var decompressedData []byte + _, err = zstd.Decompress(decompressedData, decodedData) if err != nil { return nil, fmt.Errorf("error reading decompressed data: %v", err) } - reader.Close() var result StreamInfo err = json.Unmarshal(decompressedData, &result) From 9d72211e0dff1b10a27a7485398685b45cb220f6 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:17:20 -0500 Subject: [PATCH 10/49] enable cgo on build --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index be8e1b3a..3b2969c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,11 +8,14 @@ WORKDIR /app COPY go.mod go.sum ./ # Download all dependencies. Dependencies will be cached if the go.mod and go.sum files are not changed -RUN go mod download +RUN go mod download && \ + apk add --no-cache --update gcc g++ # Copy the source code from the current directory to the Working Directory inside the container COPY . . +ENV CGO_ENABLED=1 + # test and build the app. # hadolint ignore=DL3018 RUN go test ./tests/... && go build -ldflags='-s -w' -o m3u-proxy . From 20866eefed34de9424a44dba0ec7752560958206 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:18:15 -0500 Subject: [PATCH 11/49] ignore 3018 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3b2969c3..9948dd5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ WORKDIR /app COPY go.mod go.sum ./ # Download all dependencies. Dependencies will be cached if the go.mod and go.sum files are not changed +# hadolint ignore=DL3018 RUN go mod download && \ apk add --no-cache --update gcc g++ @@ -17,7 +18,6 @@ COPY . . ENV CGO_ENABLED=1 # test and build the app. -# hadolint ignore=DL3018 RUN go test ./tests/... && go build -ldflags='-s -w' -o m3u-proxy . # End from the latest alpine image From 60b96da067128a2e3a5c8b91a3be505d251d784d Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:23:13 -0500 Subject: [PATCH 12/49] add logs to zstd compression --- store/slug.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/store/slug.go b/store/slug.go index 7e8448d2..ac3e6247 100644 --- a/store/slug.go +++ b/store/slug.go @@ -3,21 +3,31 @@ package store import ( "encoding/base64" "fmt" + "m3u-stream-merger/utils" + "os" "strings" "github.com/DataDog/zstd" "github.com/goccy/go-json" ) +var debug = os.Getenv("DEBUG") == "true" + func EncodeSlug(stream StreamInfo) string { jsonData, err := json.Marshal(stream) if err != nil { + if debug { + utils.SafeLogf("[DEBUG] Error json marshal: %v\n", err) + } return "" } var compressedData []byte - _, err = zstd.CompressLevel(compressedData, jsonData, zstd.BestCompression) + out, err := zstd.CompressLevel(compressedData, jsonData, zstd.BestCompression) if err != nil { + if debug { + utils.SafeLogf("[DEBUG] Error zstd compression (%v): %v\n", out, err) + } return "" } From 84ada3f593d89f7d67cbd4ce362c3fa016d17531 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:29:13 -0500 Subject: [PATCH 13/49] publish every pr commit workflow --- .github/workflows/pr-build.yml | 32 -------------------------------- .github/workflows/pr-publish.yml | 23 +++++++++-------------- 2 files changed, 9 insertions(+), 46 deletions(-) delete mode 100644 .github/workflows/pr-build.yml diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml deleted file mode 100644 index 16aeef1a..00000000 --- a/.github/workflows/pr-build.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Pull Request Build -on: - pull_request: - types: - - opened - - synchronize - - reopened - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Docker - Metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: | - ghcr.io/${{ github.repository }} - tags: type=raw,value=pr-${{ github.event.number }} - flavor: latest=false - - - name: Docker - Build - uses: docker/build-push-action@v5 - with: - push: false - tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/pr-publish.yml b/.github/workflows/pr-publish.yml index c2eb3574..d286e20d 100644 --- a/.github/workflows/pr-publish.yml +++ b/.github/workflows/pr-publish.yml @@ -1,12 +1,13 @@ name: Pull Request Publish on: - issue_comment: + pull_request: types: - - created + - synchronize + - opened + - reopened jobs: fetch-pr-details: - if: ${{ github.event.comment.body == '/publish' && github.event.issue.pull_request }} runs-on: ubuntu-latest outputs: sha: ${{ steps.get-sha.outputs.result }} @@ -17,18 +18,12 @@ jobs: with: result-encoding: string script: | - const prNumber = context.issue.number; - const response = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber - }); - const sha = response.data.head.sha; + const prNumber = context.payload.pull_request.number; + const sha = context.payload.pull_request.head.sha; return sha; build-and-push: needs: fetch-pr-details - if: ${{ github.event.comment.body == '/publish' && github.event.issue.pull_request }} runs-on: ubuntu-latest steps: - name: Checkout @@ -53,7 +48,7 @@ jobs: with: images: | ghcr.io/${{ github.repository }} - tags: type=raw,value=pr-${{ github.event.issue.number }} + tags: type=raw,value=pr-${{ github.event.pull_request.number }} flavor: latest=false - name: Docker - Build and Push @@ -69,8 +64,8 @@ jobs: with: header: Built and pushed Docker image for PR recreate: true - number: ${{ github.event.issue.number }} + number: ${{ github.event.pull_request.number }} message: | The Docker image for this pull request has been built and pushed to GHCR. - Image URL: `ghcr.io/${{ github.repository }}:pr-${{ github.event.issue.number }}` + Image URL: `ghcr.io/${{ github.repository }}:pr-${{ github.event.pull_request.number }}` From a6930d6d2bc03cee71b712a5033815fd1a7b13c5 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:38:15 -0500 Subject: [PATCH 14/49] use writer and reader for zstd --- store/slug.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/store/slug.go b/store/slug.go index ac3e6247..4b24cc1c 100644 --- a/store/slug.go +++ b/store/slug.go @@ -1,8 +1,10 @@ package store import ( + "bytes" "encoding/base64" "fmt" + "io" "m3u-stream-merger/utils" "os" "strings" @@ -17,21 +19,23 @@ func EncodeSlug(stream StreamInfo) string { jsonData, err := json.Marshal(stream) if err != nil { if debug { - utils.SafeLogf("[DEBUG] Error json marshal: %v\n", err) + utils.SafeLogf("[DEBUG] Error json marshal for slug: %v\n", err) } return "" } - var compressedData []byte - out, err := zstd.CompressLevel(compressedData, jsonData, zstd.BestCompression) + var compressedData bytes.Buffer + writer := zstd.NewWriterLevel(&compressedData, zstd.BestCompression) + _, err = writer.Write(jsonData) if err != nil { if debug { - utils.SafeLogf("[DEBUG] Error zstd compression (%v): %v\n", out, err) + utils.SafeLogf("[DEBUG] Error zstd compression for slug: %v\n", err) } return "" } + writer.Close() - encodedData := base64.StdEncoding.EncodeToString(compressedData) + encodedData := base64.StdEncoding.EncodeToString(compressedData.Bytes()) // 62nd char of encoding encodedData = strings.Replace(encodedData, "+", "-", -1) @@ -59,11 +63,12 @@ func DecodeSlug(encodedSlug string) (*StreamInfo, error) { return nil, fmt.Errorf("error decoding Base64 data: %v", err) } - var decompressedData []byte - _, err = zstd.Decompress(decompressedData, decodedData) + reader := zstd.NewReader(bytes.NewReader(decodedData)) + decompressedData, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf("error reading decompressed data: %v", err) } + reader.Close() var result StreamInfo err = json.Unmarshal(decompressedData, &result) From 6abbcf3a3f6f3249d31a01bd1de7e0a70a16d68f Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 21:43:29 -0500 Subject: [PATCH 15/49] add revalidating wait option --- handlers/m3u_handler.go | 2 +- store/cache.go | 18 ++++++++++-------- updater/updater.go | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index 8ebbe91b..3654920d 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -10,7 +10,7 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") w.Header().Set("Access-Control-Allow-Origin", "*") - content := store.RevalidatingGetM3U(r) + content := store.RevalidatingGetM3U(r, false) if _, err := w.Write([]byte(content)); err != nil { utils.SafeLogf("[ERROR] Failed to write response: %v\n", err) diff --git a/store/cache.go b/store/cache.go index 5165accd..7398370d 100644 --- a/store/cache.go +++ b/store/cache.go @@ -26,7 +26,7 @@ func isDebugMode() bool { return os.Getenv("DEBUG") == "true" } -func RevalidatingGetM3U(r *http.Request) string { +func RevalidatingGetM3U(r *http.Request, wait bool) string { debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Revalidating M3U cache") @@ -45,14 +45,16 @@ func RevalidatingGetM3U(r *http.Request) string { } M3uCache.RUnlock() - if debug { - utils.SafeLogln("[DEBUG] Waiting for cache revalidation to finish") - } + if wait { + if debug { + utils.SafeLogln("[DEBUG] Waiting for cache revalidation to finish") + } - // Wait for generation to finish - <-M3uCache.generationDone - if debug { - utils.SafeLogln("[DEBUG] Finished cache revalidation") + // Wait for generation to finish + <-M3uCache.generationDone + if debug { + utils.SafeLogln("[DEBUG] Finished cache revalidation") + } } return readCacheFromFile() diff --git a/updater/updater.go b/updater/updater.go index d060d97c..e95b6bde 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -106,7 +106,7 @@ func (instance *Updater) UpdateSources(ctx context.Context) { utils.SafeLogln("BASE_URL is required for CACHE_ON_SYNC to work.") } utils.SafeLogln("CACHE_ON_SYNC enabled. Building cache.") - _ = store.RevalidatingGetM3U(nil) + _ = store.RevalidatingGetM3U(nil, true) } } } From 37ac38196a1cab0d260d900d2584516a88818800 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 22:00:47 -0500 Subject: [PATCH 16/49] use memory mapping for performance --- go.mod | 5 +++++ go.sum | 4 ++++ store/cache.go | 36 ++++-------------------------------- store/parser.go | 48 ++++++++++++++++++++++++++---------------------- 4 files changed, 39 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 2e0dbbe9..ba08884d 100644 --- a/go.mod +++ b/go.mod @@ -7,3 +7,8 @@ require ( github.com/goccy/go-json v0.10.3 github.com/robfig/cron/v3 v3.0.1 ) + +require ( + github.com/edsrzf/mmap-go v1.2.0 // indirect + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect +) diff --git a/go.sum b/go.sum index 8de3bbf3..961e47a7 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,10 @@ github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= +github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/store/cache.go b/store/cache.go index 7398370d..eae3bdcd 100644 --- a/store/cache.go +++ b/store/cache.go @@ -11,14 +11,10 @@ import ( type Cache struct { sync.RWMutex - Exists bool - Revalidating bool - generationDone chan struct{} + Revalidating bool } -var M3uCache = &Cache{ - generationDone: make(chan struct{}), -} +var M3uCache = &Cache{} const cacheFilePath = "/m3u-proxy/cache.m3u" @@ -31,31 +27,12 @@ func RevalidatingGetM3U(r *http.Request, wait bool) string { if debug { utils.SafeLogln("[DEBUG] Revalidating M3U cache") } + M3uCache.RLock() - if M3uCache.Exists { + if _, err := os.Stat(cacheFilePath); err == nil { if debug { utils.SafeLogln("[DEBUG] M3U cache already exists") } - if !M3uCache.Revalidating { - if debug { - utils.SafeLogln("[DEBUG] Initiating cache revalidation") - } - - go generateM3UContent(r) - } - - M3uCache.RUnlock() - if wait { - if debug { - utils.SafeLogln("[DEBUG] Waiting for cache revalidation to finish") - } - - // Wait for generation to finish - <-M3uCache.generationDone - if debug { - utils.SafeLogln("[DEBUG] Finished cache revalidation") - } - } return readCacheFromFile() } @@ -110,12 +87,8 @@ func generateM3UContent(r *http.Request) string { utils.SafeLogf("Error writing cache to file: %v\n", err) } - M3uCache.Exists = true M3uCache.Revalidating = false - close(M3uCache.generationDone) - M3uCache.generationDone = make(chan struct{}) - return stringContent } @@ -125,7 +98,6 @@ func ClearCache() { M3uCache.Lock() defer M3uCache.Unlock() - M3uCache.Exists = false M3uCache.Revalidating = false if debug { diff --git a/store/parser.go b/store/parser.go index 2ed394d7..40e20aa3 100644 --- a/store/parser.go +++ b/store/parser.go @@ -1,12 +1,13 @@ package store import ( - "bufio" "os" "regexp" "strings" "m3u-stream-merger/utils" + + "github.com/edsrzf/mmap-go" ) func ParseStreamInfoBySlug(slug string) (*StreamInfo, error) { @@ -21,31 +22,34 @@ func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo)) error { if err != nil { return err } + defer file.Close() + + // Memory-map the file + mappedFile, err := mmap.Map(file, mmap.RDONLY, 0) + if err != nil { + return err + } + defer mappedFile.Unmap() - scanner := bufio.NewScanner(file) + // Process the file as a single large string + content := string(mappedFile) + lines := strings.Split(content, "\n") - for scanner.Scan() { - line := scanner.Text() + var currentLine string + for _, line := range lines { + line = strings.TrimSpace(line) if strings.HasPrefix(line, "#EXTINF:") { - if scanner.Scan() { - nextLine := scanner.Text() - // skip all other #EXT tags - for strings.HasPrefix(nextLine, "#") { - if scanner.Scan() { - nextLine = scanner.Text() - } else { - break - } - } - - streamInfo := parseLine(line, nextLine, m3uIndex) - - if !checkFilter(streamInfo) { - continue - } - - fn(streamInfo) + currentLine = line + } else if currentLine != "" && !strings.HasPrefix(line, "#") { + // Parse the stream info + streamInfo := parseLine(currentLine, line, m3uIndex) + currentLine = "" + + if !checkFilter(streamInfo) { + continue } + + fn(streamInfo) } } From 10a652eb4429cc0e8bfedf8b3ef352c1d4063c22 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 22:03:44 -0500 Subject: [PATCH 17/49] fix lint errcheck --- store/parser.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/store/parser.go b/store/parser.go index 40e20aa3..ada7eb4f 100644 --- a/store/parser.go +++ b/store/parser.go @@ -29,7 +29,9 @@ func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo)) error { if err != nil { return err } - defer mappedFile.Unmap() + defer func() { + _ = mappedFile.Unmap() + }() // Process the file as a single large string content := string(mappedFile) From e49c2f2f5e440495a2c11c9493e835e7f8566a52 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 22:15:14 -0500 Subject: [PATCH 18/49] rework revalidation wait --- store/cache.go | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/store/cache.go b/store/cache.go index eae3bdcd..85835710 100644 --- a/store/cache.go +++ b/store/cache.go @@ -11,10 +11,13 @@ import ( type Cache struct { sync.RWMutex - Revalidating bool + Revalidating bool + generationDone chan struct{} } -var M3uCache = &Cache{} +var M3uCache = &Cache{ + generationDone: make(chan struct{}), +} const cacheFilePath = "/m3u-proxy/cache.m3u" @@ -29,14 +32,27 @@ func RevalidatingGetM3U(r *http.Request, wait bool) string { } M3uCache.RLock() + defer M3uCache.RUnlock() + if _, err := os.Stat(cacheFilePath); err == nil { if debug { utils.SafeLogln("[DEBUG] M3U cache already exists") } + return readCacheFromFile() + } else if M3uCache.Revalidating { + if wait { + if debug { + utils.SafeLogln("[DEBUG] Revalidation already in progress, waiting...") + } + <-M3uCache.generationDone + } + + if debug { + utils.SafeLogln("[DEBUG] Revalidation finished.") + } return readCacheFromFile() } - M3uCache.RUnlock() if debug { utils.SafeLogln("[DEBUG] Existing cache not found, generating content") @@ -88,6 +104,8 @@ func generateM3UContent(r *http.Request) string { } M3uCache.Revalidating = false + close(M3uCache.generationDone) + M3uCache.generationDone = make(chan struct{}) return stringContent } @@ -123,7 +141,18 @@ func readCacheFromFile() string { } func writeCacheToFile(content string) error { - return os.WriteFile(cacheFilePath, []byte(content), 0644) + err := os.WriteFile(cacheFilePath+".new", []byte(content), 0644) + if err != nil { + return err + } + + _ = os.Remove(cacheFilePath) + err = os.Rename(cacheFilePath+".new", cacheFilePath) + if err != nil { + return err + } + + return nil } func deleteCacheFile() error { From 971a67f5b9ad375b93b18e1631da5ffc50c878fb Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 22:16:15 -0500 Subject: [PATCH 19/49] fix log for wait revalidation --- store/cache.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/store/cache.go b/store/cache.go index 85835710..ce7ba2cc 100644 --- a/store/cache.go +++ b/store/cache.go @@ -46,11 +46,12 @@ func RevalidatingGetM3U(r *http.Request, wait bool) string { utils.SafeLogln("[DEBUG] Revalidation already in progress, waiting...") } <-M3uCache.generationDone - } - if debug { - utils.SafeLogln("[DEBUG] Revalidation finished.") + if debug { + utils.SafeLogln("[DEBUG] Revalidation finished.") + } } + return readCacheFromFile() } From 388d45dde6677e58bb3e485c46f2f528f3da521a Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 22:31:52 -0500 Subject: [PATCH 20/49] remove revalidation flag and rely on mutex --- store/cache.go | 37 +++++++++---------------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/store/cache.go b/store/cache.go index ce7ba2cc..ef19f7ff 100644 --- a/store/cache.go +++ b/store/cache.go @@ -11,13 +11,9 @@ import ( type Cache struct { sync.RWMutex - Revalidating bool - generationDone chan struct{} } -var M3uCache = &Cache{ - generationDone: make(chan struct{}), -} +var M3uCache = &Cache{} const cacheFilePath = "/m3u-proxy/cache.m3u" @@ -25,33 +21,24 @@ func isDebugMode() bool { return os.Getenv("DEBUG") == "true" } -func RevalidatingGetM3U(r *http.Request, wait bool) string { +func RevalidatingGetM3U(r *http.Request, force bool) string { debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Revalidating M3U cache") } + if force { + return generateM3UContent(r) + } + M3uCache.RLock() - defer M3uCache.RUnlock() if _, err := os.Stat(cacheFilePath); err == nil { if debug { utils.SafeLogln("[DEBUG] M3U cache already exists") } - return readCacheFromFile() - } else if M3uCache.Revalidating { - if wait { - if debug { - utils.SafeLogln("[DEBUG] Revalidation already in progress, waiting...") - } - <-M3uCache.generationDone - - if debug { - utils.SafeLogln("[DEBUG] Revalidation finished.") - } - } - + M3uCache.RUnlock() return readCacheFromFile() } @@ -59,6 +46,8 @@ func RevalidatingGetM3U(r *http.Request, wait bool) string { utils.SafeLogln("[DEBUG] Existing cache not found, generating content") } + M3uCache.RUnlock() + return generateM3UContent(r) } @@ -66,8 +55,6 @@ func generateM3UContent(r *http.Request) string { M3uCache.Lock() defer M3uCache.Unlock() - M3uCache.Revalidating = true - debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Regenerating M3U cache in the background") @@ -104,10 +91,6 @@ func generateM3UContent(r *http.Request) string { utils.SafeLogf("Error writing cache to file: %v\n", err) } - M3uCache.Revalidating = false - close(M3uCache.generationDone) - M3uCache.generationDone = make(chan struct{}) - return stringContent } @@ -117,8 +100,6 @@ func ClearCache() { M3uCache.Lock() defer M3uCache.Unlock() - M3uCache.Revalidating = false - if debug { utils.SafeLogln("[DEBUG] Clearing memory and disk M3U cache.") } From 01f135e410bfc1f2f5ec3bddf8d67c929f22bfb0 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 22:38:38 -0500 Subject: [PATCH 21/49] move download sources to tmp and cache to data --- store/cache.go | 10 ++++++++-- utils/m3u_path.go | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/store/cache.go b/store/cache.go index ef19f7ff..3c627e7e 100644 --- a/store/cache.go +++ b/store/cache.go @@ -5,6 +5,7 @@ import ( "m3u-stream-merger/utils" "net/http" "os" + "path/filepath" "strings" "sync" ) @@ -15,7 +16,7 @@ type Cache struct { var M3uCache = &Cache{} -const cacheFilePath = "/m3u-proxy/cache.m3u" +const cacheFilePath = "/m3u-proxy/data/cache.m3u" func isDebugMode() bool { return os.Getenv("DEBUG") == "true" @@ -123,7 +124,12 @@ func readCacheFromFile() string { } func writeCacheToFile(content string) error { - err := os.WriteFile(cacheFilePath+".new", []byte(content), 0644) + err := os.MkdirAll(filepath.Dir(cacheFilePath), os.ModePerm) + if err != nil { + return fmt.Errorf("Error creating directories for data path: %v", err) + } + + err = os.WriteFile(cacheFilePath+".new", []byte(content), 0644) if err != nil { return err } diff --git a/utils/m3u_path.go b/utils/m3u_path.go index 59985989..6316a49c 100644 --- a/utils/m3u_path.go +++ b/utils/m3u_path.go @@ -3,7 +3,7 @@ package utils import "fmt" func GetM3UFilePathByIndex(m3uIndex int) string { - m3uFile := fmt.Sprintf("/m3u-proxy/sources/%d.m3u", m3uIndex+1) + m3uFile := fmt.Sprintf("/tmp/m3u-proxy/sources/%d.m3u", m3uIndex+1) return m3uFile } From c44bcdcdaac7cc2346789ee3f2fac74c7c251bb1 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 22:49:34 -0500 Subject: [PATCH 22/49] only use title and urls in json + readme edit --- README.md | 27 ++++++++------------------- store/types.go | 10 +++++----- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index efcda5dc..ed717f2d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,12 @@ Streamline your IPTV experience by consolidating multiple M3U playlists into a s Uses the channel title or `tvg-name` (as fallback) to merge multiple identical channels into one. This is not an xTeVe/Threadfin replacement but is often used with it. > [!IMPORTANT] -> All versions after `0.10.0` will require an external Redis/Valkey instance. The SQLite database within the data folder will not be used going forward. For data persistence, refer to the [Redis](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/) docs. The sample `docker-compose.yml` below has also been modified to include Redis. +> Starting `0.16.0`, Redis will be removed as a dependency. There will be no databases required for the proxy from this version moving forward. +> Migrating to `0.16.0` is as easy as removing the Redis container from your compose file. +> Due to a major change on how data is being processed, any Redis persistence cannot be migrated over and a sync from the original M3U sources will be required. + +> [!NOTICE] +> All versions after `0.10.0` until `0.15.2` requires an external Redis/Valkey instance. For data persistence, refer to the [Redis](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/) docs. > To see the README of a specific version, navigate to the specific tag of the desired version (e.g. [`0.10.0`](https://github.com/sonroyaalmerol/m3u-stream-merger-proxy/tree/0.10.0)). ## How It Works @@ -53,7 +58,6 @@ Deploy with ease using the provided `docker-compose.yml`: ```yaml -version: '3' services: m3u-stream-merger-proxy: image: sonroyaalmerol/m3u-stream-merger-proxy:latest @@ -72,20 +76,9 @@ services: - M3U_MAX_CONCURRENCY_2=1 - M3U_URL_X= restart: always - depends_on: - - redis - redis: - image: redis - restart: always - healthcheck: - test: ["CMD-SHELL", "redis-cli ping | grep PONG"] - interval: 1s - timeout: 3s - retries: 5 - # Redis persistence is OPTIONAL. This will allow you to reuse the database across restarts. - # command: redis-server --save 60 1 + # [OPTIONAL] Cache persistence: This will allow you to reuse the M3U cache across container recreates. # volumes: - # - ./data:/data + # - ./data:/m3u-proxy/data ``` Access the generated M3U playlist at `http://:8080/playlist.m3u`. @@ -103,12 +96,8 @@ Access the generated M3U playlist at `http://:8080/playlist.m3u`. | MAX_RETRIES | Set max number of retries (loop) across all M3Us while streaming. 0 to never stop retrying (beware of throttling from provider). | 5 | Any integer greater than or equal 0 | | RETRY_WAIT | Set a wait time before retrying (looping) across all M3Us on stream initialization error. | 0 | Any integer greater than or equal 0 | | STREAM_TIMEOUT | Set timeout duration in seconds of retrying on error before a stream is considered down. | 3 | Any positive integer greater than 0 | -| REDIS_ADDR | Set Redis server address | N/A | e.g. localhost:6379 | -| REDIS_PASS | Set Redis server password | N/A | Any string | -| REDIS_DB | Set Redis server database to be used | 0 | 0 to 15 | | SORTING_KEY | Set tag to be used for sorting the stream list | tvg-id | tvg-id, tvg-chno | | USER_AGENT | Set the User-Agent of HTTP requests. | IPTV Smarters/1.0.3 (iPad; iOS 16.6.1; Scale/2.00) | Any valid user agent | -| ~~LOAD_BALANCING_MODE~~ (removed on version 0.10.0) | Set load balancing algorithm to a specific mode | brute-force | brute-force/round-robin | | PARSER_WORKERS | Set number of workers to spawn for M3U parsing. | 5 | Any positive integer | | BUFFER_MB | Set buffer size in mb. | 0 (no buffer) | Any positive integer | | INCLUDE_GROUPS_1, INCLUDE_GROUPS_2, INCLUDE_GROUPS_X | Set channels to include based on groups (Takes precedence over EXCLUDE_GROUPS_X) | N/A | Go regexp | diff --git a/store/types.go b/store/types.go index 2dff4c27..b967f59a 100644 --- a/store/types.go +++ b/store/types.go @@ -1,11 +1,11 @@ package store type StreamInfo struct { - Slug string `json:"slug"` + Slug string `json:"-"` Title string `json:"title"` - TvgID string `json:"tvg_id"` - TvgChNo string `json:"tvg_chno"` - LogoURL string `json:"logo_url"` - Group string `json:"group_name"` + TvgID string `json:"-"` + TvgChNo string `json:"-"` + LogoURL string `json:"-"` + Group string `json:"-"` URLs map[int]string `json:"urls"` } From 031b59819c87031ddaed544469fd158239e16400 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 22:54:21 -0500 Subject: [PATCH 23/49] fix readme --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ed717f2d..467547e5 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,10 @@ Streamline your IPTV experience by consolidating multiple M3U playlists into a s Uses the channel title or `tvg-name` (as fallback) to merge multiple identical channels into one. This is not an xTeVe/Threadfin replacement but is often used with it. > [!IMPORTANT] -> Starting `0.16.0`, Redis will be removed as a dependency. There will be no databases required for the proxy from this version moving forward. +> Starting `0.16.0`, Redis is **removed** as a dependency. There will be **no** databases required for the proxy from this version moving forward. > Migrating to `0.16.0` is as easy as removing the Redis container from your compose file. > Due to a major change on how data is being processed, any Redis persistence cannot be migrated over and a sync from the original M3U sources will be required. -> [!NOTICE] -> All versions after `0.10.0` until `0.15.2` requires an external Redis/Valkey instance. For data persistence, refer to the [Redis](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/) docs. -> To see the README of a specific version, navigate to the specific tag of the desired version (e.g. [`0.10.0`](https://github.com/sonroyaalmerol/m3u-stream-merger-proxy/tree/0.10.0)). - ## How It Works 1. **Initialization and M3U Playlist Consolidation:** @@ -85,6 +81,10 @@ Access the generated M3U playlist at `http://:8080/playlist.m3u`. ## Configuration +> [!NOTE] +> This configuration list only applies to the latest release version. +> To see the README of a specific version, navigate to the specific tag of the desired version (e.g. [`0.10.0`](https://github.com/sonroyaalmerol/m3u-stream-merger-proxy/tree/0.10.0)). + | ENV VAR | Description | Default Value | Possible Values | |-----------------------------|----------------------------------------------------------|---------------|------------------------------------------------| | PORT | Set listening port of service inside the container. | 8080 | Any valid port | From f3effff207b076487c35adad185d6cc981c22363 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 23:13:31 -0500 Subject: [PATCH 24/49] separate goroutines for each m3u index --- store/parser.go | 1 + store/streams.go | 43 ++++++++++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/store/parser.go b/store/parser.go index ada7eb4f..108fa840 100644 --- a/store/parser.go +++ b/store/parser.go @@ -38,6 +38,7 @@ func M3UScanner(m3uIndex int, fn func(streamInfo StreamInfo)) error { lines := strings.Split(content, "\n") var currentLine string + for _, line := range lines { line = strings.TrimSpace(line) if strings.HasPrefix(line, "#EXTINF:") { diff --git a/store/streams.go b/store/streams.go index 4b7f7979..d79afe74 100644 --- a/store/streams.go +++ b/store/streams.go @@ -5,6 +5,7 @@ import ( "m3u-stream-merger/utils" "os" "sort" + "sync" ) func GetStreamBySlug(slug string) (StreamInfo, error) { @@ -19,30 +20,38 @@ func GetStreamBySlug(slug string) (StreamInfo, error) { func GetStreams() []StreamInfo { var ( debug = os.Getenv("DEBUG") == "true" - streams = make(map[string]StreamInfo) // Map to store unique streams by title - result = make([]StreamInfo, 0) // Slice to store final results + result = make([]StreamInfo, 0) // Slice to store final results + streams sync.Map ) + var wg sync.WaitGroup for _, m3uIndex := range utils.GetM3UIndexes() { - err := M3UScanner(m3uIndex, func(streamInfo StreamInfo) { - // Check uniqueness and update if necessary - if existingStream, exists := streams[streamInfo.Title]; exists { - for idx, url := range streamInfo.URLs { - existingStream.URLs[idx] = url + wg.Add(1) + go func(m3uIndex int) { + defer wg.Done() + + err := M3UScanner(m3uIndex, func(streamInfo StreamInfo) { + // Check uniqueness and update if necessary + if existingStream, exists := streams.Load(streamInfo.Title); exists { + for idx, url := range streamInfo.URLs { + existingStream.(StreamInfo).URLs[idx] = url + } + streams.Store(streamInfo.Title, existingStream) + } else { + streams.Store(streamInfo.Title, streamInfo) } - streams[streamInfo.Title] = existingStream - } else { - streams[streamInfo.Title] = streamInfo + }) + if err != nil && debug { + utils.SafeLogf("error getting streams: %v\n", err) } - }) - if err != nil && debug { - utils.SafeLogf("error getting streams: %v\n", err) - } + }(m3uIndex) } + wg.Wait() - for _, stream := range streams { - result = append(result, stream) - } + streams.Range(func(key, value any) bool { + result = append(result, value.(StreamInfo)) + return true + }) sortStreams(result) From a6fb94e7db91efeaaeb6165948b5c2447bdc70c3 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 23:21:34 -0500 Subject: [PATCH 25/49] prevent blocking http access while loading proxy --- store/cache.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/store/cache.go b/store/cache.go index 3c627e7e..f7f99af9 100644 --- a/store/cache.go +++ b/store/cache.go @@ -32,14 +32,11 @@ func RevalidatingGetM3U(r *http.Request, force bool) string { return generateM3UContent(r) } - M3uCache.RLock() - if _, err := os.Stat(cacheFilePath); err == nil { if debug { utils.SafeLogln("[DEBUG] M3U cache already exists") } - M3uCache.RUnlock() return readCacheFromFile() } @@ -47,7 +44,9 @@ func RevalidatingGetM3U(r *http.Request, force bool) string { utils.SafeLogln("[DEBUG] Existing cache not found, generating content") } - M3uCache.RUnlock() + if !M3uCache.TryRLock() { + return readCacheFromFile() + } return generateM3UContent(r) } From fde2befb7efe926ed684e64e6bfd29e44e4ef7f3 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 23:31:48 -0500 Subject: [PATCH 26/49] fix blocking revalidation --- store/cache.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/store/cache.go b/store/cache.go index f7f99af9..7a02a225 100644 --- a/store/cache.go +++ b/store/cache.go @@ -32,23 +32,18 @@ func RevalidatingGetM3U(r *http.Request, force bool) string { return generateM3UContent(r) } - if _, err := os.Stat(cacheFilePath); err == nil { + if _, err := os.Stat(cacheFilePath); err != nil { if debug { - utils.SafeLogln("[DEBUG] M3U cache already exists") + utils.SafeLogln("[DEBUG] Existing cache not found, generating content") + } + if M3uCache.TryLock() { + go func() { + _ = generateM3UContent(r) + }() } - - return readCacheFromFile() - } - - if debug { - utils.SafeLogln("[DEBUG] Existing cache not found, generating content") - } - - if !M3uCache.TryRLock() { - return readCacheFromFile() } - return generateM3UContent(r) + return readCacheFromFile() } func generateM3UContent(r *http.Request) string { From 4a54c11a4be94889bc33e77b8e70fe9bdc11f1f6 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 23:36:14 -0500 Subject: [PATCH 27/49] fix blocking test --- tests/proxy_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/proxy_test.go b/tests/proxy_test.go index 7adbda77..439a2afa 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -22,11 +22,13 @@ func TestStreamHandler(t *testing.T) { t.Errorf("Downloader returned error: %v", err) } - streams := store.GetStreams() - m3uReq := httptest.NewRequest("GET", "/playlist.m3u", nil) m3uW := httptest.NewRecorder() + _ = store.RevalidatingGetM3U(m3uReq, true) + + streams := store.GetStreams() + func() { handlers.M3UHandler(m3uW, m3uReq) }() From 6254f9ced554ce55126160da9dcc8eac3df70ecb Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sat, 16 Nov 2024 23:44:29 -0500 Subject: [PATCH 28/49] only generate slugs on url generate --- store/parser.go | 6 ------ store/streams.go | 9 +++++---- store/types.go | 1 - 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/store/parser.go b/store/parser.go index 108fa840..724e5501 100644 --- a/store/parser.go +++ b/store/parser.go @@ -115,11 +115,5 @@ func parseLine(line string, nextLine string, m3uIndex int) StreamInfo { currentStream.Title = utils.TvgNameParser(strings.TrimSpace(lineCommaSplit[1])) } - currentStream.Slug = EncodeSlug(currentStream) - - if debug { - utils.SafeLogf("[DEBUG] Generated slug: %s\n", currentStream.Slug) - } - return currentStream } diff --git a/store/streams.go b/store/streams.go index d79afe74..baa736bd 100644 --- a/store/streams.go +++ b/store/streams.go @@ -49,7 +49,8 @@ func GetStreams() []StreamInfo { wg.Wait() streams.Range(func(key, value any) bool { - result = append(result, value.(StreamInfo)) + stream := value.(StreamInfo) + result = append(result, stream) return true }) @@ -69,12 +70,12 @@ func GenerateStreamURL(baseUrl string, stream StreamInfo) string { ext, err := utils.GetFileExtensionFromUrl(srcUrl) if err != nil { - return fmt.Sprintf("%s/proxy/%s/%s", baseUrl, subPath, stream.Slug) + return fmt.Sprintf("%s/proxy/%s/%s", baseUrl, subPath, EncodeSlug(stream)) } - return fmt.Sprintf("%s/proxy/%s/%s%s", baseUrl, subPath, stream.Slug, ext) + return fmt.Sprintf("%s/proxy/%s/%s%s", baseUrl, subPath, EncodeSlug(stream), ext) } - return fmt.Sprintf("%s/proxy/stream/%s", baseUrl, stream.Slug) + return fmt.Sprintf("%s/proxy/stream/%s", baseUrl, EncodeSlug(stream)) } func sortStreams(s []StreamInfo) { diff --git a/store/types.go b/store/types.go index b967f59a..b6663009 100644 --- a/store/types.go +++ b/store/types.go @@ -1,7 +1,6 @@ package store type StreamInfo struct { - Slug string `json:"-"` Title string `json:"title"` TvgID string `json:"-"` TvgChNo string `json:"-"` From 4319637e7c76dd1f939e170796c338b1505253bb Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:07:17 -0500 Subject: [PATCH 29/49] use channels for m3u generate --- handlers/m3u_handler.go | 20 ++++++++- store/cache.go | 97 ++++++++++++++++++++++++----------------- 2 files changed, 75 insertions(+), 42 deletions(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index 3654920d..ae6740db 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -1,18 +1,34 @@ package handlers import ( + "fmt" "m3u-stream-merger/store" "m3u-stream-merger/utils" "net/http" + "os" ) func M3UHandler(w http.ResponseWriter, r *http.Request) { + debug := os.Getenv("DEBUG") == "true" + w.Header().Set("Content-Type", "text/plain") w.Header().Set("Access-Control-Allow-Origin", "*") content := store.RevalidatingGetM3U(r, false) - if _, err := w.Write([]byte(content)); err != nil { - utils.SafeLogf("[ERROR] Failed to write response: %v\n", err) + for { + data, ok := <-content + if !ok { + break + } + + _, err := fmt.Fprintf(w, data) + if err != nil { + if debug { + utils.SafeLogf("[DEBUG] Error writing http response: %v\n", err) + } + } } + + w.WriteHeader(http.StatusOK) } diff --git a/store/cache.go b/store/cache.go index 7a02a225..49157b67 100644 --- a/store/cache.go +++ b/store/cache.go @@ -22,31 +22,24 @@ func isDebugMode() bool { return os.Getenv("DEBUG") == "true" } -func RevalidatingGetM3U(r *http.Request, force bool) string { +func RevalidatingGetM3U(r *http.Request, force bool) chan string { debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Revalidating M3U cache") } - if force { - return generateM3UContent(r) - } - - if _, err := os.Stat(cacheFilePath); err != nil { - if debug { + if _, err := os.Stat(cacheFilePath); err != nil || force { + if debug && !force { utils.SafeLogln("[DEBUG] Existing cache not found, generating content") } - if M3uCache.TryLock() { - go func() { - _ = generateM3UContent(r) - }() - } + + return generateM3UContent(r) } return readCacheFromFile() } -func generateM3UContent(r *http.Request) string { +func generateM3UContent(r *http.Request) chan string { M3uCache.Lock() defer M3uCache.Unlock() @@ -60,33 +53,52 @@ func generateM3UContent(r *http.Request) string { utils.SafeLogf("[DEBUG] Base URL set to %s\n", baseURL) } - var content strings.Builder - content.WriteString("#EXTM3U\n") + contentStream := make(chan string) + egressStream := make(chan string) - streams := GetStreams() - for _, stream := range streams { - if len(stream.URLs) == 0 { - continue - } + go func() { + var content strings.Builder + content.WriteString("#EXTM3U\n") + egressStream <- "#EXTM3U\n" - if debug { - utils.SafeLogf("[DEBUG] Processing stream with TVG ID: %s\n", stream.TvgID) + for { + data, ok := <-contentStream + if !ok { + if debug { + utils.SafeLogln("[DEBUG] Finished generating M3U content") + } + close(egressStream) + break + } + + egressStream <- data + content.WriteString(data) } - content.WriteString(formatStreamEntry(baseURL, stream)) - } + stringContent := content.String() - if debug { - utils.SafeLogln("[DEBUG] Finished generating M3U content") - } + if err := writeCacheToFile(stringContent); err != nil { + utils.SafeLogf("Error writing cache to file: %v\n", err) + } + }() - stringContent := content.String() + go func() { + streams := GetStreams() + for _, stream := range streams { + if len(stream.URLs) == 0 { + continue + } - if err := writeCacheToFile(stringContent); err != nil { - utils.SafeLogf("Error writing cache to file: %v\n", err) - } + if debug { + utils.SafeLogf("[DEBUG] Processing stream with TVG ID: %s\n", stream.TvgID) + } - return stringContent + contentStream <- formatStreamEntry(baseURL, stream) + } + close(contentStream) + }() + + return egressStream } func ClearCache() { @@ -103,18 +115,23 @@ func ClearCache() { } } -func readCacheFromFile() string { +func readCacheFromFile() chan string { debug := isDebugMode() - data, err := os.ReadFile(cacheFilePath) - if err != nil { - if debug { - utils.SafeLogf("[DEBUG] Cache file reading failed: %v\n", err) + dataChan := make(chan string) + go func() { + data, err := os.ReadFile(cacheFilePath) + if err != nil { + if debug { + utils.SafeLogf("[DEBUG] Cache file reading failed: %v\n", err) + } + + dataChan <- "#EXTM3U\n" } + dataChan <- string(data) + }() - return "#EXTM3U\n" - } - return string(data) + return dataChan } func writeCacheToFile(content string) error { From b5bdd800ad2e4f93205a1eb94f3c4c509d37fbc2 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:08:56 -0500 Subject: [PATCH 30/49] use fprint instead --- handlers/m3u_handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index ae6740db..e17d9cf1 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -22,7 +22,7 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { break } - _, err := fmt.Fprintf(w, data) + _, err := fmt.Fprint(w, data) if err != nil { if debug { utils.SafeLogf("[DEBUG] Error writing http response: %v\n", err) From 1cd8400d818f875585d076bdf7f201c6a48dbca3 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:12:44 -0500 Subject: [PATCH 31/49] use http writer response --- handlers/m3u_handler.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index e17d9cf1..09519200 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -1,7 +1,6 @@ package handlers import ( - "fmt" "m3u-stream-merger/store" "m3u-stream-merger/utils" "net/http" @@ -22,7 +21,7 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { break } - _, err := fmt.Fprint(w, data) + _, err := w.Write([]byte(data)) if err != nil { if debug { utils.SafeLogf("[DEBUG] Error writing http response: %v\n", err) From ed63e49397089938fe6d76b7521b259ff2a90604 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:17:55 -0500 Subject: [PATCH 32/49] use performant zstd --- Dockerfile | 6 +----- go.mod | 8 +++----- go.sum | 4 ++-- store/slug.go | 17 ++++++++++++++--- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9948dd5f..f22dcea3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,15 +8,11 @@ WORKDIR /app COPY go.mod go.sum ./ # Download all dependencies. Dependencies will be cached if the go.mod and go.sum files are not changed -# hadolint ignore=DL3018 -RUN go mod download && \ - apk add --no-cache --update gcc g++ +RUN go mod download # Copy the source code from the current directory to the Working Directory inside the container COPY . . -ENV CGO_ENABLED=1 - # test and build the app. RUN go test ./tests/... && go build -ldflags='-s -w' -o m3u-proxy . diff --git a/go.mod b/go.mod index ba08884d..5de008d2 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,10 @@ module m3u-stream-merger go 1.23.0 require ( - github.com/DataDog/zstd v1.5.6 + github.com/edsrzf/mmap-go v1.2.0 github.com/goccy/go-json v0.10.3 + github.com/klauspost/compress v1.17.11 github.com/robfig/cron/v3 v3.0.1 ) -require ( - github.com/edsrzf/mmap-go v1.2.0 // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect -) +require golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect diff --git a/go.sum b/go.sum index 961e47a7..e257f56d 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ -github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= -github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84= github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= diff --git a/store/slug.go b/store/slug.go index 4b24cc1c..b0a979a8 100644 --- a/store/slug.go +++ b/store/slug.go @@ -9,8 +9,8 @@ import ( "os" "strings" - "github.com/DataDog/zstd" "github.com/goccy/go-json" + "github.com/klauspost/compress/zstd" ) var debug = os.Getenv("DEBUG") == "true" @@ -25,7 +25,14 @@ func EncodeSlug(stream StreamInfo) string { } var compressedData bytes.Buffer - writer := zstd.NewWriterLevel(&compressedData, zstd.BestCompression) + writer, err := zstd.NewWriter(&compressedData) + if err != nil { + if debug { + utils.SafeLogf("[DEBUG] Error zstd compression for slug: %v\n", err) + } + return "" + } + _, err = writer.Write(jsonData) if err != nil { if debug { @@ -63,7 +70,11 @@ func DecodeSlug(encodedSlug string) (*StreamInfo, error) { return nil, fmt.Errorf("error decoding Base64 data: %v", err) } - reader := zstd.NewReader(bytes.NewReader(decodedData)) + reader, err := zstd.NewReader(bytes.NewReader(decodedData)) + if err != nil { + return nil, fmt.Errorf("error reading decompressed data: %v", err) + } + decompressedData, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf("error reading decompressed data: %v", err) From c2a094485560cdfb937ee93b17ce5cf7bb8c73b9 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:23:51 -0500 Subject: [PATCH 33/49] add read lock to revalidation --- store/cache.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/store/cache.go b/store/cache.go index 49157b67..aaa5852f 100644 --- a/store/cache.go +++ b/store/cache.go @@ -28,14 +28,18 @@ func RevalidatingGetM3U(r *http.Request, force bool) chan string { utils.SafeLogln("[DEBUG] Revalidating M3U cache") } + M3uCache.RLock() + if _, err := os.Stat(cacheFilePath); err != nil || force { if debug && !force { utils.SafeLogln("[DEBUG] Existing cache not found, generating content") } + M3uCache.RUnlock() return generateM3UContent(r) } + M3uCache.RUnlock() return readCacheFromFile() } From 8aa243ca6b25c79e4b0e0b208fdce9a1e310e027 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:31:22 -0500 Subject: [PATCH 34/49] fix cache on sync --- store/cache.go | 2 +- tests/proxy_test.go | 2 -- updater/updater.go | 7 ++++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/store/cache.go b/store/cache.go index aaa5852f..fcaeb05c 100644 --- a/store/cache.go +++ b/store/cache.go @@ -94,7 +94,7 @@ func generateM3UContent(r *http.Request) chan string { } if debug { - utils.SafeLogf("[DEBUG] Processing stream with TVG ID: %s\n", stream.TvgID) + utils.SafeLogf("[DEBUG] Processing stream title: %s\n", stream.Title) } contentStream <- formatStreamEntry(baseURL, stream) diff --git a/tests/proxy_test.go b/tests/proxy_test.go index 439a2afa..fe254e76 100644 --- a/tests/proxy_test.go +++ b/tests/proxy_test.go @@ -25,8 +25,6 @@ func TestStreamHandler(t *testing.T) { m3uReq := httptest.NewRequest("GET", "/playlist.m3u", nil) m3uW := httptest.NewRecorder() - _ = store.RevalidatingGetM3U(m3uReq, true) - streams := store.GetStreams() func() { diff --git a/updater/updater.go b/updater/updater.go index e95b6bde..4a8cbadc 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -106,7 +106,12 @@ func (instance *Updater) UpdateSources(ctx context.Context) { utils.SafeLogln("BASE_URL is required for CACHE_ON_SYNC to work.") } utils.SafeLogln("CACHE_ON_SYNC enabled. Building cache.") - _ = store.RevalidatingGetM3U(nil, true) + for { + _, ok := <-store.RevalidatingGetM3U(nil, true) + if !ok { + break + } + } } } } From 79936e6a248b99f75b5a5c32e2e7656d053785fd Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:49:52 -0500 Subject: [PATCH 35/49] add try rlock and read cache --- handlers/m3u_handler.go | 5 +++-- store/cache.go | 49 ++++++++++++++++++++++------------------- updater/updater.go | 5 ++++- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index 09519200..d37154f3 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -13,10 +13,11 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") w.Header().Set("Access-Control-Allow-Origin", "*") - content := store.RevalidatingGetM3U(r, false) + contentStream := make(chan string) + go store.RevalidatingGetM3U(r, contentStream, false) for { - data, ok := <-content + data, ok := <-contentStream if !ok { break } diff --git a/store/cache.go b/store/cache.go index fcaeb05c..9ee82979 100644 --- a/store/cache.go +++ b/store/cache.go @@ -22,28 +22,32 @@ func isDebugMode() bool { return os.Getenv("DEBUG") == "true" } -func RevalidatingGetM3U(r *http.Request, force bool) chan string { +func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Revalidating M3U cache") } - M3uCache.RLock() + if !M3uCache.TryRLock() && !force { + readCacheFromFile(egressStream) + return + } + M3uCache.RLock() if _, err := os.Stat(cacheFilePath); err != nil || force { if debug && !force { utils.SafeLogln("[DEBUG] Existing cache not found, generating content") } M3uCache.RUnlock() - return generateM3UContent(r) + generateM3UContent(r, egressStream) } - M3uCache.RUnlock() - return readCacheFromFile() + + readCacheFromFile(egressStream) } -func generateM3UContent(r *http.Request) chan string { +func generateM3UContent(r *http.Request, egressStream chan string) { M3uCache.Lock() defer M3uCache.Unlock() @@ -58,11 +62,12 @@ func generateM3UContent(r *http.Request) chan string { } contentStream := make(chan string) - egressStream := make(chan string) + + if err := writeCacheToFile(egressStream); err != nil { + utils.SafeLogf("Error writing cache to file: %v\n", err) + } go func() { - var content strings.Builder - content.WriteString("#EXTM3U\n") egressStream <- "#EXTM3U\n" for { @@ -76,13 +81,6 @@ func generateM3UContent(r *http.Request) chan string { } egressStream <- data - content.WriteString(data) - } - - stringContent := content.String() - - if err := writeCacheToFile(stringContent); err != nil { - utils.SafeLogf("Error writing cache to file: %v\n", err) } }() @@ -102,7 +100,7 @@ func generateM3UContent(r *http.Request) chan string { close(contentStream) }() - return egressStream + return } func ClearCache() { @@ -119,10 +117,9 @@ func ClearCache() { } } -func readCacheFromFile() chan string { +func readCacheFromFile(dataChan chan string) { debug := isDebugMode() - dataChan := make(chan string) go func() { data, err := os.ReadFile(cacheFilePath) if err != nil { @@ -134,21 +131,27 @@ func readCacheFromFile() chan string { } dataChan <- string(data) }() - - return dataChan } -func writeCacheToFile(content string) error { +func writeCacheToFile(content chan string) error { err := os.MkdirAll(filepath.Dir(cacheFilePath), os.ModePerm) if err != nil { return fmt.Errorf("Error creating directories for data path: %v", err) } - err = os.WriteFile(cacheFilePath+".new", []byte(content), 0644) + file, err := os.OpenFile(cacheFilePath+".new", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err } + for { + data, ok := <-content + if !ok { + break + } + file.WriteString(data) + } + _ = os.Remove(cacheFilePath) err = os.Rename(cacheFilePath+".new", cacheFilePath) if err != nil { diff --git a/updater/updater.go b/updater/updater.go index 4a8cbadc..2a2667ac 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -106,8 +106,11 @@ func (instance *Updater) UpdateSources(ctx context.Context) { utils.SafeLogln("BASE_URL is required for CACHE_ON_SYNC to work.") } utils.SafeLogln("CACHE_ON_SYNC enabled. Building cache.") + + contentStream := make(chan string) + go store.RevalidatingGetM3U(nil, contentStream, true) for { - _, ok := <-store.RevalidatingGetM3U(nil, true) + _, ok := <-contentStream if !ok { break } From c55fc17a7037b2926a41260d4ca214235fd37530 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:51:07 -0500 Subject: [PATCH 36/49] separate goroutine for cache to file --- store/cache.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/store/cache.go b/store/cache.go index 9ee82979..8f7ea3e0 100644 --- a/store/cache.go +++ b/store/cache.go @@ -63,9 +63,11 @@ func generateM3UContent(r *http.Request, egressStream chan string) { contentStream := make(chan string) - if err := writeCacheToFile(egressStream); err != nil { - utils.SafeLogf("Error writing cache to file: %v\n", err) - } + go func() { + if err := writeCacheToFile(egressStream); err != nil { + utils.SafeLogf("Error writing cache to file: %v\n", err) + } + }() go func() { egressStream <- "#EXTM3U\n" From 8715983f2a09f2bc9f06ba1132f20eeb51b9a85a Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:52:39 -0500 Subject: [PATCH 37/49] fix linting error --- store/cache.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/store/cache.go b/store/cache.go index 8f7ea3e0..953420ef 100644 --- a/store/cache.go +++ b/store/cache.go @@ -101,8 +101,6 @@ func generateM3UContent(r *http.Request, egressStream chan string) { } close(contentStream) }() - - return } func ClearCache() { @@ -151,7 +149,7 @@ func writeCacheToFile(content chan string) error { if !ok { break } - file.WriteString(data) + _, _ = file.WriteString(data) } _ = os.Remove(cacheFilePath) From 20b15fc664e900c57b49c7f3b14853f2f1fc6328 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 00:58:19 -0500 Subject: [PATCH 38/49] fix blocking issue --- store/cache.go | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/store/cache.go b/store/cache.go index 953420ef..8afae7c8 100644 --- a/store/cache.go +++ b/store/cache.go @@ -40,10 +40,12 @@ func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { } M3uCache.RUnlock() + generateM3UContent(r, egressStream) + return } - M3uCache.RUnlock() + M3uCache.RUnlock() readCacheFromFile(egressStream) } @@ -63,11 +65,9 @@ func generateM3UContent(r *http.Request, egressStream chan string) { contentStream := make(chan string) - go func() { - if err := writeCacheToFile(egressStream); err != nil { - utils.SafeLogf("Error writing cache to file: %v\n", err) - } - }() + if err := writeCacheToFile(egressStream); err != nil { + utils.SafeLogf("Error writing cache to file: %v\n", err) + } go func() { egressStream <- "#EXTM3U\n" @@ -144,19 +144,20 @@ func writeCacheToFile(content chan string) error { return err } - for { - data, ok := <-content - if !ok { - break + go func() { + for { + data, ok := <-content + if !ok { + break + } + _, _ = file.WriteString(data) } - _, _ = file.WriteString(data) - } - _ = os.Remove(cacheFilePath) - err = os.Rename(cacheFilePath+".new", cacheFilePath) - if err != nil { - return err - } + _ = os.Remove(cacheFilePath) + _ = os.Rename(cacheFilePath+".new", cacheFilePath) + + _ = file.Close() + }() return nil } From 9ac0eedf5fc76b934a521a8a372aeb0aa8012112 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:03:01 -0500 Subject: [PATCH 39/49] close datachan on file read --- store/cache.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/store/cache.go b/store/cache.go index 8afae7c8..fb72d2d6 100644 --- a/store/cache.go +++ b/store/cache.go @@ -128,8 +128,11 @@ func readCacheFromFile(dataChan chan string) { } dataChan <- "#EXTM3U\n" + } else { + dataChan <- string(data) } - dataChan <- string(data) + + close(dataChan) }() } From ea4fe2dcbdae8d314b2532e9cc30cc7cc2e0871c Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:07:57 -0500 Subject: [PATCH 40/49] use pointers for egress chan --- handlers/m3u_handler.go | 2 +- store/cache.go | 24 ++++++++++++------------ updater/updater.go | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index d37154f3..e4ce921d 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -15,7 +15,7 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { contentStream := make(chan string) - go store.RevalidatingGetM3U(r, contentStream, false) + go store.RevalidatingGetM3U(r, &contentStream, false) for { data, ok := <-contentStream if !ok { diff --git a/store/cache.go b/store/cache.go index fb72d2d6..4d6ee2f7 100644 --- a/store/cache.go +++ b/store/cache.go @@ -22,7 +22,7 @@ func isDebugMode() bool { return os.Getenv("DEBUG") == "true" } -func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { +func RevalidatingGetM3U(r *http.Request, egressStream *chan string, force bool) { debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Revalidating M3U cache") @@ -49,7 +49,7 @@ func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { readCacheFromFile(egressStream) } -func generateM3UContent(r *http.Request, egressStream chan string) { +func generateM3UContent(r *http.Request, egressStream *chan string) { M3uCache.Lock() defer M3uCache.Unlock() @@ -70,7 +70,7 @@ func generateM3UContent(r *http.Request, egressStream chan string) { } go func() { - egressStream <- "#EXTM3U\n" + *egressStream <- "#EXTM3U\n" for { data, ok := <-contentStream @@ -78,11 +78,11 @@ func generateM3UContent(r *http.Request, egressStream chan string) { if debug { utils.SafeLogln("[DEBUG] Finished generating M3U content") } - close(egressStream) - break + close(*egressStream) + return } - egressStream <- data + *egressStream <- data } }() @@ -117,7 +117,7 @@ func ClearCache() { } } -func readCacheFromFile(dataChan chan string) { +func readCacheFromFile(dataChan *chan string) { debug := isDebugMode() go func() { @@ -127,16 +127,16 @@ func readCacheFromFile(dataChan chan string) { utils.SafeLogf("[DEBUG] Cache file reading failed: %v\n", err) } - dataChan <- "#EXTM3U\n" + *dataChan <- "#EXTM3U\n" } else { - dataChan <- string(data) + *dataChan <- string(data) } - close(dataChan) + close(*dataChan) }() } -func writeCacheToFile(content chan string) error { +func writeCacheToFile(content *chan string) error { err := os.MkdirAll(filepath.Dir(cacheFilePath), os.ModePerm) if err != nil { return fmt.Errorf("Error creating directories for data path: %v", err) @@ -149,7 +149,7 @@ func writeCacheToFile(content chan string) error { go func() { for { - data, ok := <-content + data, ok := <-*content if !ok { break } diff --git a/updater/updater.go b/updater/updater.go index 2a2667ac..56325fa1 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -108,7 +108,7 @@ func (instance *Updater) UpdateSources(ctx context.Context) { utils.SafeLogln("CACHE_ON_SYNC enabled. Building cache.") contentStream := make(chan string) - go store.RevalidatingGetM3U(nil, contentStream, true) + go store.RevalidatingGetM3U(nil, &contentStream, true) for { _, ok := <-contentStream if !ok { From 198450aa4d4ee1bf15a41dc5aaf21db66d1a695c Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:13:33 -0500 Subject: [PATCH 41/49] remove trylock --- handlers/m3u_handler.go | 7 +++---- store/cache.go | 27 +++++++++++---------------- updater/updater.go | 2 +- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index e4ce921d..68b71d12 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -15,11 +15,12 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { contentStream := make(chan string) - go store.RevalidatingGetM3U(r, &contentStream, false) + go store.RevalidatingGetM3U(r, contentStream, false) for { data, ok := <-contentStream if !ok { - break + w.WriteHeader(http.StatusOK) + return } _, err := w.Write([]byte(data)) @@ -29,6 +30,4 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { } } } - - w.WriteHeader(http.StatusOK) } diff --git a/store/cache.go b/store/cache.go index 4d6ee2f7..7ae0c89a 100644 --- a/store/cache.go +++ b/store/cache.go @@ -22,17 +22,12 @@ func isDebugMode() bool { return os.Getenv("DEBUG") == "true" } -func RevalidatingGetM3U(r *http.Request, egressStream *chan string, force bool) { +func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Revalidating M3U cache") } - if !M3uCache.TryRLock() && !force { - readCacheFromFile(egressStream) - return - } - M3uCache.RLock() if _, err := os.Stat(cacheFilePath); err != nil || force { if debug && !force { @@ -49,7 +44,7 @@ func RevalidatingGetM3U(r *http.Request, egressStream *chan string, force bool) readCacheFromFile(egressStream) } -func generateM3UContent(r *http.Request, egressStream *chan string) { +func generateM3UContent(r *http.Request, egressStream chan string) { M3uCache.Lock() defer M3uCache.Unlock() @@ -70,7 +65,7 @@ func generateM3UContent(r *http.Request, egressStream *chan string) { } go func() { - *egressStream <- "#EXTM3U\n" + egressStream <- "#EXTM3U\n" for { data, ok := <-contentStream @@ -78,11 +73,11 @@ func generateM3UContent(r *http.Request, egressStream *chan string) { if debug { utils.SafeLogln("[DEBUG] Finished generating M3U content") } - close(*egressStream) + close(egressStream) return } - *egressStream <- data + egressStream <- data } }() @@ -117,7 +112,7 @@ func ClearCache() { } } -func readCacheFromFile(dataChan *chan string) { +func readCacheFromFile(dataChan chan string) { debug := isDebugMode() go func() { @@ -127,16 +122,16 @@ func readCacheFromFile(dataChan *chan string) { utils.SafeLogf("[DEBUG] Cache file reading failed: %v\n", err) } - *dataChan <- "#EXTM3U\n" + dataChan <- "#EXTM3U\n" } else { - *dataChan <- string(data) + dataChan <- string(data) } - close(*dataChan) + close(dataChan) }() } -func writeCacheToFile(content *chan string) error { +func writeCacheToFile(content chan string) error { err := os.MkdirAll(filepath.Dir(cacheFilePath), os.ModePerm) if err != nil { return fmt.Errorf("Error creating directories for data path: %v", err) @@ -149,7 +144,7 @@ func writeCacheToFile(content *chan string) error { go func() { for { - data, ok := <-*content + data, ok := <-content if !ok { break } diff --git a/updater/updater.go b/updater/updater.go index 56325fa1..2a2667ac 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -108,7 +108,7 @@ func (instance *Updater) UpdateSources(ctx context.Context) { utils.SafeLogln("CACHE_ON_SYNC enabled. Building cache.") contentStream := make(chan string) - go store.RevalidatingGetM3U(nil, &contentStream, true) + go store.RevalidatingGetM3U(nil, contentStream, true) for { _, ok := <-contentStream if !ok { From d9a11933cc2007feca4f13ad33698b7c1bd009c8 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:17:26 -0500 Subject: [PATCH 42/49] fix mutex lock for write --- store/cache.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/store/cache.go b/store/cache.go index 7ae0c89a..1194e22e 100644 --- a/store/cache.go +++ b/store/cache.go @@ -45,9 +45,6 @@ func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { } func generateM3UContent(r *http.Request, egressStream chan string) { - M3uCache.Lock() - defer M3uCache.Unlock() - debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Regenerating M3U cache in the background") @@ -82,6 +79,9 @@ func generateM3UContent(r *http.Request, egressStream chan string) { }() go func() { + M3uCache.Lock() + defer M3uCache.Unlock() + streams := GetStreams() for _, stream := range streams { if len(stream.URLs) == 0 { From ff26ea206ad8ca9a6133a7ac890580e9fc90e9b8 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:26:09 -0500 Subject: [PATCH 43/49] fix #EXTM3U header --- handlers/m3u_handler.go | 1 - store/cache.go | 11 ++--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index 68b71d12..a3518bde 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -19,7 +19,6 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { for { data, ok := <-contentStream if !ok { - w.WriteHeader(http.StatusOK) return } diff --git a/store/cache.go b/store/cache.go index 1194e22e..2b5dd256 100644 --- a/store/cache.go +++ b/store/cache.go @@ -11,7 +11,7 @@ import ( ) type Cache struct { - sync.RWMutex + sync.Mutex } var M3uCache = &Cache{} @@ -28,19 +28,15 @@ func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { utils.SafeLogln("[DEBUG] Revalidating M3U cache") } - M3uCache.RLock() if _, err := os.Stat(cacheFilePath); err != nil || force { if debug && !force { utils.SafeLogln("[DEBUG] Existing cache not found, generating content") } - M3uCache.RUnlock() - generateM3UContent(r, egressStream) return } - M3uCache.RUnlock() readCacheFromFile(egressStream) } @@ -62,8 +58,6 @@ func generateM3UContent(r *http.Request, egressStream chan string) { } go func() { - egressStream <- "#EXTM3U\n" - for { data, ok := <-contentStream if !ok { @@ -83,6 +77,7 @@ func generateM3UContent(r *http.Request, egressStream chan string) { defer M3uCache.Unlock() streams := GetStreams() + contentStream <- "#EXTM3U\n" for _, stream := range streams { if len(stream.URLs) == 0 { continue @@ -121,8 +116,6 @@ func readCacheFromFile(dataChan chan string) { if debug { utils.SafeLogf("[DEBUG] Cache file reading failed: %v\n", err) } - - dataChan <- "#EXTM3U\n" } else { dataChan <- string(data) } From cb18cce643b25b341661ca1f0bb1fcd75c086384 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:29:08 -0500 Subject: [PATCH 44/49] fix ext header for cache from file --- store/cache.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/store/cache.go b/store/cache.go index 2b5dd256..493962c7 100644 --- a/store/cache.go +++ b/store/cache.go @@ -111,6 +111,8 @@ func readCacheFromFile(dataChan chan string) { debug := isDebugMode() go func() { + dataChan <- "#EXTM3U\n" + data, err := os.ReadFile(cacheFilePath) if err != nil { if debug { From e2bed72a635b3f0063354132dd0208934d007183 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:39:02 -0500 Subject: [PATCH 45/49] fix write cache to file --- store/cache.go | 66 +++++++++++++------------------------------------- 1 file changed, 17 insertions(+), 49 deletions(-) diff --git a/store/cache.go b/store/cache.go index 493962c7..aa5d63b5 100644 --- a/store/cache.go +++ b/store/cache.go @@ -51,33 +51,15 @@ func generateM3UContent(r *http.Request, egressStream chan string) { utils.SafeLogf("[DEBUG] Base URL set to %s\n", baseURL) } - contentStream := make(chan string) - - if err := writeCacheToFile(egressStream); err != nil { - utils.SafeLogf("Error writing cache to file: %v\n", err) - } - - go func() { - for { - data, ok := <-contentStream - if !ok { - if debug { - utils.SafeLogln("[DEBUG] Finished generating M3U content") - } - close(egressStream) - return - } - - egressStream <- data - } - }() + go writeCacheToFile(egressStream) go func() { M3uCache.Lock() defer M3uCache.Unlock() streams := GetStreams() - contentStream <- "#EXTM3U\n" + egressStream <- "#EXTM3U\n" + for _, stream := range streams { if len(stream.URLs) == 0 { continue @@ -87,9 +69,9 @@ func generateM3UContent(r *http.Request, egressStream chan string) { utils.SafeLogf("[DEBUG] Processing stream title: %s\n", stream.Title) } - contentStream <- formatStreamEntry(baseURL, stream) + egressStream <- formatStreamEntry(baseURL, stream) } - close(contentStream) + close(egressStream) }() } @@ -111,8 +93,6 @@ func readCacheFromFile(dataChan chan string) { debug := isDebugMode() go func() { - dataChan <- "#EXTM3U\n" - data, err := os.ReadFile(cacheFilePath) if err != nil { if debug { @@ -126,33 +106,21 @@ func readCacheFromFile(dataChan chan string) { }() } -func writeCacheToFile(content chan string) error { - err := os.MkdirAll(filepath.Dir(cacheFilePath), os.ModePerm) - if err != nil { - return fmt.Errorf("Error creating directories for data path: %v", err) - } - - file, err := os.OpenFile(cacheFilePath+".new", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } +func writeCacheToFile(dataChan chan string) { + var content strings.Builder - go func() { - for { - data, ok := <-content - if !ok { - break - } - _, _ = file.WriteString(data) + for { + data, ok := <-dataChan + if !ok { + break } + content.WriteString(data) + } - _ = os.Remove(cacheFilePath) - _ = os.Rename(cacheFilePath+".new", cacheFilePath) - - _ = file.Close() - }() - - return nil + _ = os.MkdirAll(filepath.Dir(cacheFilePath), os.ModePerm) + _ = os.WriteFile(cacheFilePath+".new", []byte(content.String()), 0644) + _ = os.Remove(cacheFilePath) + _ = os.Rename(cacheFilePath+".new", cacheFilePath) } func deleteCacheFile() error { From 385d6c6bad0439982edf5ceefbc844d495d48b63 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:47:12 -0500 Subject: [PATCH 46/49] revert back to non-channel handling --- handlers/m3u_handler.go | 19 +++------ store/cache.go | 90 +++++++++++++++++++++-------------------- updater/updater.go | 9 +---- 3 files changed, 53 insertions(+), 65 deletions(-) diff --git a/handlers/m3u_handler.go b/handlers/m3u_handler.go index a3518bde..618a4607 100644 --- a/handlers/m3u_handler.go +++ b/handlers/m3u_handler.go @@ -13,20 +13,11 @@ func M3UHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") w.Header().Set("Access-Control-Allow-Origin", "*") - contentStream := make(chan string) - - go store.RevalidatingGetM3U(r, contentStream, false) - for { - data, ok := <-contentStream - if !ok { - return - } - - _, err := w.Write([]byte(data)) - if err != nil { - if debug { - utils.SafeLogf("[DEBUG] Error writing http response: %v\n", err) - } + content := store.RevalidatingGetM3U(r, false) + _, err := w.Write([]byte(content)) + if err != nil { + if debug { + utils.SafeLogf("[DEBUG] Error writing http response: %v\n", err) } } } diff --git a/store/cache.go b/store/cache.go index aa5d63b5..bd385ade 100644 --- a/store/cache.go +++ b/store/cache.go @@ -22,7 +22,7 @@ func isDebugMode() bool { return os.Getenv("DEBUG") == "true" } -func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { +func RevalidatingGetM3U(r *http.Request, force bool) string { debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Revalidating M3U cache") @@ -33,14 +33,13 @@ func RevalidatingGetM3U(r *http.Request, egressStream chan string, force bool) { utils.SafeLogln("[DEBUG] Existing cache not found, generating content") } - generateM3UContent(r, egressStream) - return + return generateM3UContent(r) } - readCacheFromFile(egressStream) + return readCacheFromFile() } -func generateM3UContent(r *http.Request, egressStream chan string) { +func generateM3UContent(r *http.Request) string { debug := isDebugMode() if debug { utils.SafeLogln("[DEBUG] Regenerating M3U cache in the background") @@ -51,28 +50,32 @@ func generateM3UContent(r *http.Request, egressStream chan string) { utils.SafeLogf("[DEBUG] Base URL set to %s\n", baseURL) } - go writeCacheToFile(egressStream) + var content strings.Builder - go func() { - M3uCache.Lock() - defer M3uCache.Unlock() + M3uCache.Lock() + defer M3uCache.Unlock() - streams := GetStreams() - egressStream <- "#EXTM3U\n" + streams := GetStreams() - for _, stream := range streams { - if len(stream.URLs) == 0 { - continue - } + content.WriteString("#EXTM3U\n") - if debug { - utils.SafeLogf("[DEBUG] Processing stream title: %s\n", stream.Title) - } + for _, stream := range streams { + if len(stream.URLs) == 0 { + continue + } - egressStream <- formatStreamEntry(baseURL, stream) + if debug { + utils.SafeLogf("[DEBUG] Processing stream title: %s\n", stream.Title) } - close(egressStream) - }() + + content.WriteString(formatStreamEntry(baseURL, stream)) + } + + if err := writeCacheToFile(content.String()); err != nil { + utils.SafeLogf("[DEBUG] Error writing cache to file: %v\n", err) + } + + return content.String() } func ClearCache() { @@ -89,38 +92,39 @@ func ClearCache() { } } -func readCacheFromFile(dataChan chan string) { +func readCacheFromFile() string { debug := isDebugMode() - go func() { - data, err := os.ReadFile(cacheFilePath) - if err != nil { - if debug { - utils.SafeLogf("[DEBUG] Cache file reading failed: %v\n", err) - } - } else { - dataChan <- string(data) + data, err := os.ReadFile(cacheFilePath) + if err != nil { + if debug { + utils.SafeLogf("[DEBUG] Cache file reading failed: %v\n", err) } - close(dataChan) - }() + return "#EXTM3U\n" + } + + return string(data) } -func writeCacheToFile(dataChan chan string) { - var content strings.Builder +func writeCacheToFile(content string) error { + err := os.MkdirAll(filepath.Dir(cacheFilePath), os.ModePerm) + if err != nil { + return err + } - for { - data, ok := <-dataChan - if !ok { - break - } - content.WriteString(data) + err = os.WriteFile(cacheFilePath+".new", []byte(content), 0644) + if err != nil { + return err } - _ = os.MkdirAll(filepath.Dir(cacheFilePath), os.ModePerm) - _ = os.WriteFile(cacheFilePath+".new", []byte(content.String()), 0644) _ = os.Remove(cacheFilePath) - _ = os.Rename(cacheFilePath+".new", cacheFilePath) + + err = os.Rename(cacheFilePath+".new", cacheFilePath) + if err != nil { + return err + } + return nil } func deleteCacheFile() error { diff --git a/updater/updater.go b/updater/updater.go index 2a2667ac..5128f026 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -107,14 +107,7 @@ func (instance *Updater) UpdateSources(ctx context.Context) { } utils.SafeLogln("CACHE_ON_SYNC enabled. Building cache.") - contentStream := make(chan string) - go store.RevalidatingGetM3U(nil, contentStream, true) - for { - _, ok := <-contentStream - if !ok { - break - } - } + _ = store.RevalidatingGetM3U(nil, true) } } } From 144e74f14213c6eca97e1cb181871364787629a5 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:51:37 -0500 Subject: [PATCH 47/49] shorter subpath --- main.go | 4 ++-- store/streams.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 44a0cf8a..471310ce 100644 --- a/main.go +++ b/main.go @@ -39,14 +39,14 @@ func main() { http.HandleFunc("/playlist.m3u", func(w http.ResponseWriter, r *http.Request) { handlers.M3UHandler(w, r) }) - http.HandleFunc("/proxy/", func(w http.ResponseWriter, r *http.Request) { + http.HandleFunc("/p/", func(w http.ResponseWriter, r *http.Request) { handlers.StreamHandler(w, r, cm) }) // Start the server utils.SafeLogln(fmt.Sprintf("Server is running on port %s...", os.Getenv("PORT"))) utils.SafeLogln("Playlist Endpoint is running (`/playlist.m3u`)") - utils.SafeLogln("Stream Endpoint is running (`/proxy/{originalBasePath}/{streamID}.{fileExt}`)") + utils.SafeLogln("Stream Endpoint is running (`/p/{originalBasePath}/{streamID}.{fileExt}`)") err = http.ListenAndServe(fmt.Sprintf(":%s", os.Getenv("PORT")), nil) if err != nil { utils.SafeLogFatalf("HTTP server error: %v", err) diff --git a/store/streams.go b/store/streams.go index baa736bd..c5e00d1d 100644 --- a/store/streams.go +++ b/store/streams.go @@ -70,12 +70,12 @@ func GenerateStreamURL(baseUrl string, stream StreamInfo) string { ext, err := utils.GetFileExtensionFromUrl(srcUrl) if err != nil { - return fmt.Sprintf("%s/proxy/%s/%s", baseUrl, subPath, EncodeSlug(stream)) + return fmt.Sprintf("%s/p/%s/%s", baseUrl, subPath, EncodeSlug(stream)) } - return fmt.Sprintf("%s/proxy/%s/%s%s", baseUrl, subPath, EncodeSlug(stream), ext) + return fmt.Sprintf("%s/p/%s/%s%s", baseUrl, subPath, EncodeSlug(stream), ext) } - return fmt.Sprintf("%s/proxy/stream/%s", baseUrl, EncodeSlug(stream)) + return fmt.Sprintf("%s/p/stream/%s", baseUrl, EncodeSlug(stream)) } func sortStreams(s []StreamInfo) { From f22ccfbbc7227633721802161c5e601f305373c3 Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:54:02 -0500 Subject: [PATCH 48/49] add log on finish m3u generate --- store/cache.go | 2 ++ updater/updater.go | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/store/cache.go b/store/cache.go index bd385ade..7930302a 100644 --- a/store/cache.go +++ b/store/cache.go @@ -75,6 +75,8 @@ func generateM3UContent(r *http.Request) string { utils.SafeLogf("[DEBUG] Error writing cache to file: %v\n", err) } + utils.SafeLogln("Background process: Finished building M3U content.") + return content.String() } diff --git a/updater/updater.go b/updater/updater.go index 5128f026..e95b6bde 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -106,7 +106,6 @@ func (instance *Updater) UpdateSources(ctx context.Context) { utils.SafeLogln("BASE_URL is required for CACHE_ON_SYNC to work.") } utils.SafeLogln("CACHE_ON_SYNC enabled. Building cache.") - _ = store.RevalidatingGetM3U(nil, true) } } From 18b673c9a6c12b6d6044ff71b9bcf66dd0cef34f Mon Sep 17 00:00:00 2001 From: Son Roy Almerol Date: Sun, 17 Nov 2024 01:57:37 -0500 Subject: [PATCH 49/49] use RWMutex for concurrenct --- store/concurrency.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/store/concurrency.go b/store/concurrency.go index e4a3f03d..6666a6be 100644 --- a/store/concurrency.go +++ b/store/concurrency.go @@ -9,7 +9,7 @@ import ( ) type ConcurrencyManager struct { - mu sync.Mutex + mu sync.RWMutex count map[int]int } @@ -32,8 +32,8 @@ func (cm *ConcurrencyManager) Decrement(m3uIndex int) { } func (cm *ConcurrencyManager) GetCount(m3uIndex int) int { - cm.mu.Lock() - defer cm.mu.Unlock() + cm.mu.RLock() + defer cm.mu.RUnlock() return cm.count[m3uIndex] }