From 69ae19e512ff6d42899068c602b449c7c4122306 Mon Sep 17 00:00:00 2001 From: Viktor Varland Date: Thu, 27 Mar 2025 19:41:58 +0100 Subject: [PATCH] refactor: extract download to package dl --- .gitignore | 3 ++ internal/dl/dl.go | 84 +++++++++++++++++++++++++++++++++++ internal/dl/yt.go | 59 ------------------------- main.go | 110 ++++++---------------------------------------- 4 files changed, 101 insertions(+), 155 deletions(-) create mode 100644 .gitignore create mode 100644 internal/dl/dl.go delete mode 100644 internal/dl/yt.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..529a74e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.direnv +config.toml +subs-opml.xml diff --git a/internal/dl/dl.go b/internal/dl/dl.go new file mode 100644 index 0000000..12e57d4 --- /dev/null +++ b/internal/dl/dl.go @@ -0,0 +1,84 @@ +package dl + +import ( + "bufio" + "log" + "net/url" + "os/exec" + "path/filepath" + "sync" +) + +type Download struct { + Url *url.URL + OutDir string + AfterDate string + Name string +} + +func Get(d Download) { + output := filepath.Join(d.Name, "%(upload_date>%Y)s/%(upload_date)s-%(title)s-%(id)s.%(ext)s") + archive := filepath.Join(d.OutDir, d.Name, "archive.txt") + outdir := filepath.Join(d.OutDir, d.Name) + + cmd := exec.Command("yt-dlp", + d.Url.String(), + "--no-simulate", + "--no-progress", + "--sleep-interval", "1", + "--sleep-subtitles", "3", + "--sleep-requests", "1", + "--prefer-free-formats", + "--write-subs", + "--no-write-automatic-subs", + "--sub-langs", "en", + "--dateafter", d.AfterDate, + "--paths", outdir, + "--output", output, + "--download-archive", archive, + "--break-on-existing", + "--restrict-filenames", + ) + + stdout, err := cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + log.Fatal(err) + } + + log.Printf("[%s] running yt-dlp for: %s", d.Name, d.Url) + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + log.Printf("[%s] %s\n", d.Name, scanner.Text()) + } + }() + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + log.Printf("[%s] %s\n", d.Name, scanner.Text()) + } + }() + + err = cmd.Start() + if err != nil { + log.Fatal(err) + } + + wg.Wait() + err = cmd.Wait() + + if err != nil { + log.Printf("Error: %s\n", err) + } +} diff --git a/internal/dl/yt.go b/internal/dl/yt.go deleted file mode 100644 index 3030358..0000000 --- a/internal/dl/yt.go +++ /dev/null @@ -1,59 +0,0 @@ -package dl - -import ( - "bufio" - "log" - "os/exec" - "sync" -) - -func List(channelId string) { - cmd := exec.Command("yt-dlp", - "--skip-download", - "--force-write-archive", - "--download-archive", "test.txt", - channelId, - ) - - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - stderr, err := cmd.StderrPipe() - if err != nil { - log.Fatal(err) - } - - log.Printf("Running yt-dlp for: %s", d.Url) - - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - log.Printf("%s\n", scanner.Text()) - } - }() - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - log.Printf("%s\n", scanner.Text()) - } - }() - - err = cmd.Start() - if err != nil { - log.Fatal(err) - } - - wg.Wait() - err = cmd.Wait() - - if err != nil { - log.Printf("Error: %s\n", err) - } -} diff --git a/main.go b/main.go index d96f82e..7149d48 100644 --- a/main.go +++ b/main.go @@ -1,123 +1,37 @@ package main import ( - "bufio" "fmt" - "log" "net/url" "os" - "os/exec" - "path/filepath" "strings" - "sync" + "git.meatbag.se/varl/subsyt/internal/dl" "git.meatbag.se/varl/subsyt/internal/opml" toml "github.com/pelletier/go-toml/v2" ) -func check(e error) { - if e != nil { - panic(e) - } -} - type Provider struct { Url string } -type Channel struct { - Name string - Id string - Provider string -} - type Config struct { Out_dir string Provider map[string]Provider } -type Download struct { - Url *url.URL - OutDir string - AfterDate string - Name string -} - -func dl(d Download) { - - output := filepath.Join(d.OutDir, d.Name, "%(upload_date>%Y)s/%(upload_date)s-%(title)s-%(id)s.%(ext)s") - archive := filepath.Join(d.OutDir, d.Name, "archive.txt") - outdir := filepath.Join(d.OutDir, d.Name) - - cmd := exec.Command("yt-dlp", - d.Url.String(), - "--no-simulate", - "--no-progress", - "--sleep-interval", "1", - "--sleep-subtitles", "1", - "--sleep-requests", "0", - "--prefer-free-formats", - "--write-subs", - "--no-write-automatic-subs", - "--sub-langs", "en", - "--dateafter", d.AfterDate, - "--paths", outdir, - "--output", output, - "--download-archive", archive, - "--break-on-existing", - ) - - stdout, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - stderr, err := cmd.StderrPipe() - if err != nil { - log.Fatal(err) - } - - log.Printf("Running yt-dlp for: %s", d.Url) - - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - log.Printf("[%s] %s\n", d.Name, scanner.Text()) - } - }() - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - log.Printf("[%s] %s\n", d.Name, scanner.Text()) - } - }() - - err = cmd.Start() - if err != nil { - log.Fatal(err) - } - - wg.Wait() - err = cmd.Wait() - - if err != nil { - log.Printf("Error: %s\n", err) - } -} - func main() { data, err := os.ReadFile("./config.toml") - check(err) + if err != nil { + panic(err) + } cfg := Config{} err = toml.Unmarshal(data, &cfg) - check(err) + if err != nil { + panic(err) + } pUrl := cfg.Provider["youtube"].Url @@ -133,13 +47,17 @@ func main() { for _, outline := range outlines.Outlines { curl := strings.TrimPrefix(outline.XmlUrl, "/feed/") furl, err := url.JoinPath(pUrl, curl) - check(err) + if err != nil { + panic(err) + } fmt.Printf("%s - %s\n", outline.Text, furl) fullUrl, err := url.Parse(furl) - check(err) + if err != nil { + panic(err) + } - dl(Download{ + dl.Get(dl.Download{ Name: outline.Title, Url: fullUrl, OutDir: cfg.Out_dir,