diff --git a/README.md b/README.md
index 4b477fa..c9564b4 100644
--- a/README.md
+++ b/README.md
@@ -11,6 +11,8 @@ out_dir = "./vids" # path to archive vids
[provider]
[provider.youtube]
cmd = "./yt-dlp" # path to yt-dlp binary
+quality = "res:1080" # set the preferred quality
+output_path_template = "" # yt-dlp output template
url = "https://www.youtube.com" # full yt url
throttle = 1 # throttle yt request
range = "1:5:1" # [START][:STOP][:STEP]
@@ -42,6 +44,9 @@ opml_file = "./opml.xml" # the opml file to use
## Cookies
+> [!CRITICAL]
+> Your account **MAY** be banned when using cookies !
+
E.g. from Chromium:
```
diff --git a/internal/config/config.go b/internal/config/config.go
index 05b285d..7bc5ea9 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -8,14 +8,16 @@ import (
)
type Provider struct {
- Url string
- Throttle int
- Range string
- After_date string
- Cmd string
- Cookies bool
- Cookies_file string
- Opml_file string
+ Url string
+ Throttle int
+ Range string
+ After_date string
+ Cmd string
+ Cookies bool
+ Cookies_file string
+ Opml_file string
+ Quality string
+ Output_path_template string
}
type Config struct {
@@ -37,7 +39,7 @@ func Load(filepath string) (Config, error) {
panic(err)
}
- log.Printf("Loaded config:")
+ log.Println("Loaded config:")
log.Printf("%+v\n", cfg)
return cfg, err
diff --git a/internal/dl/dl.go b/internal/dl/dl.go
index b24e2ab..d4d83ae 100644
--- a/internal/dl/dl.go
+++ b/internal/dl/dl.go
@@ -2,8 +2,11 @@ package dl
import (
"bufio"
+ "io"
"log"
+ "net/http"
"net/url"
+ "os"
"os/exec"
"path/filepath"
"strconv"
@@ -20,10 +23,9 @@ type Download struct {
DryRun bool
}
-func Get(d Download, p config.Provider) {
- output := filepath.Join("%(channel)s-%(title)s-%(id)s.%(ext)s")
- archive := filepath.Join(d.OutDir, d.Name, "archive.txt")
- outdir := filepath.Join(d.OutDir, d.Name)
+func Youtube(d Download, p config.Provider) {
+ archive := filepath.Join(d.OutDir, "archive.txt")
+ outdir := d.OutDir
curl := strings.TrimPrefix(d.Url, "/feed/")
furl, err := url.JoinPath(p.Url, curl, "videos")
@@ -43,22 +45,26 @@ func Get(d Download, p config.Provider) {
"--sleep-interval", throttle,
"--sleep-subtitles", throttle,
"--sleep-requests", throttle,
+ "--format-sort", p.Quality,
"--prefer-free-formats",
"--write-subs",
"--no-write-automatic-subs",
"--sub-langs", "en",
"--paths", outdir,
- "--output", output,
+ "--output", p.Output_path_template,
"--download-archive", archive,
"--break-on-existing",
"--playlist-items", p.Range,
"--restrict-filenames",
"--embed-metadata",
+ "--write-thumbnail",
+ "--write-info-json",
+ "--convert-thumbnails", "jpg",
}
if d.DryRun == true {
args = append(args, "--simulate")
- log.Printf("/!\\ DRY RUN ENABLED /!\\")
+ log.Println("/!\\ DRY RUN ENABLED /!\\")
} else {
args = append(args, "--no-simulate")
}
@@ -83,7 +89,7 @@ func Get(d Download, p config.Provider) {
log.Fatal(err)
}
- log.Printf("[%s] running yt-dlp for: %s", d.Name, d.Url)
+ log.Printf("[%s] running yt-dlp for: %s\n", d.OutDir, d.Url)
var wg sync.WaitGroup
wg.Add(2)
@@ -92,7 +98,7 @@ func Get(d Download, p config.Provider) {
defer wg.Done()
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
- log.Printf("[%s] %s\n", d.Name, scanner.Text())
+ log.Printf("[%s] %s\n", d.OutDir, scanner.Text())
}
}()
@@ -100,7 +106,7 @@ func Get(d Download, p config.Provider) {
defer wg.Done()
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
- log.Printf("[%s] %s\n", d.Name, scanner.Text())
+ log.Printf("[%s] %s\n", d.OutDir, scanner.Text())
}
}()
@@ -116,3 +122,30 @@ func Get(d Download, p config.Provider) {
log.Printf("Error: %s\n", err)
}
}
+
+func Fetch(d Download) {
+ // Create output directory if it doesn't exist
+ if err := os.MkdirAll(d.OutDir, 0755); err != nil {
+ }
+
+ outputPath := filepath.Join(d.OutDir, d.Name)
+
+ out, err := os.Create(outputPath)
+ if err != nil {
+ log.Printf("failed to create '%s'\n", outputPath)
+ return
+ }
+ defer out.Close()
+
+ resp, err := http.Get(d.Url)
+ if err != nil {
+ log.Printf("failed to download '%s'\n", d.Url)
+ return
+ }
+ defer resp.Body.Close()
+
+ _, err = io.Copy(out, resp.Body)
+ if err != nil {
+ log.Printf("failed to write file")
+ }
+}
diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go
new file mode 100644
index 0000000..d9415e6
--- /dev/null
+++ b/internal/metadata/metadata.go
@@ -0,0 +1,128 @@
+package metadata
+
+import (
+ "fmt"
+ "log"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+
+ "git.meatbag.se/varl/subsyt/internal/dl"
+ "git.meatbag.se/varl/subsyt/internal/models"
+ "git.meatbag.se/varl/subsyt/internal/nfo"
+)
+
+func findFiles(scanPath string, ext string) ([]string, error) {
+ var result []string
+
+ err := filepath.Walk(scanPath, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.IsDir() && strings.HasSuffix(path, ext) {
+ result = append(result, path)
+ }
+ return nil
+ })
+
+ if err != nil {
+ return nil, fmt.Errorf("error walking directory: %w", err)
+ }
+
+ return result, nil
+}
+
+func Generate(outDir string, title string, dryRun bool) {
+ showDir := filepath.Join(outDir, title)
+ log.Printf("Writing NFO's for %s\n", showDir)
+
+ if dryRun {
+ return
+ }
+
+ infojsons, err := findFiles(showDir, ".info.json")
+ if err != nil {
+ panic(err)
+ }
+
+ show := regexp.MustCompile("s(NA)")
+ season := regexp.MustCompile(`s\d\d\d\d`)
+
+ for index, path := range infojsons {
+ log.Println(index, path)
+ switch {
+ case show.MatchString(path):
+ show := models.LoadShow(path)
+ nfo.WriteShowInfo(show, filepath.Join(showDir, "tvshow.nfo"))
+ showBanner(show, showDir)
+ case season.MatchString(path):
+ ep := models.LoadEpisode(path)
+ nfo.WriteEpisodeNFO(ep, path)
+ default:
+ log.Printf("no match for '%s'\n", path)
+ }
+
+ os.Remove(path)
+ }
+
+ images, err := findFiles(showDir, ".jpg")
+ if err != nil {
+ panic(err)
+ }
+
+ for index, path := range images {
+ log.Println(index, path)
+ switch {
+ case show.MatchString(path):
+ showPoster(path, showDir)
+ case season.MatchString(path):
+ episodeImage(path)
+ default:
+ log.Printf("no match for '%s'\n", path)
+ }
+ }
+
+ del := filepath.Join(showDir, "sNA")
+ log.Printf("removing '%s'\n", del)
+ err = os.RemoveAll(del)
+ if err != nil {
+ log.Println("failed to remove", err)
+ }
+}
+
+func episodeImage(path string) {
+ if strings.Contains(path, "-thumb") {
+ log.Printf("thumbnail detected '%s'\n", path)
+ return
+ }
+ thumb := strings.Replace(path, ".jpg", "-thumb.jpg", 1)
+ log.Printf("renaming thumbnail from '%s' to '%s'\n", path, thumb)
+ err := os.Rename(path, thumb)
+ if err != nil {
+ log.Printf("failed to rename '%s' to '%s\n'", path, thumb)
+ }
+}
+
+func showPoster(path string, show_dir string) {
+ poster := filepath.Join(show_dir, "poster.jpg")
+ log.Printf("renaming show image from '%s' to '%s'\n", path, poster)
+ err := os.Rename(path, poster)
+ if err != nil {
+ log.Printf("failed to rename '%s' to '%s\n'", path, poster)
+ }
+}
+
+func showBanner(show models.Show, showDir string) {
+ for index, thumb := range show.Thumbnails {
+ log.Println(index, thumb)
+ if thumb.Id == "banner_uncropped" {
+ log.Println("found banner candidate")
+ dl.Fetch(dl.Download{
+ Url: thumb.Url,
+ OutDir: showDir,
+ Name: "banner.jpg",
+ })
+ }
+ }
+}
diff --git a/internal/models/episode.go b/internal/models/episode.go
new file mode 100644
index 0000000..3c16760
--- /dev/null
+++ b/internal/models/episode.go
@@ -0,0 +1,73 @@
+package models
+
+import (
+ "encoding/json"
+ "encoding/xml"
+ "fmt"
+ "os"
+ "time"
+)
+
+// Target Episode XML structure:
+//
+//
+//
+// #{safe(metadata["title"])}
+// #{safe(metadata["uploader"])}
+// #{safe(metadata["id"])}
+// #{safe(metadata["description"])}
+// #{safe(upload_date)}
+// #{safe(season)}
+// #{episode}
+// YouTube
+//
+
+type Episode struct {
+ XMLName xml.Name `xml:"episodedetails"`
+ Title string `json:"title" xml:"title"`
+ ShowTitle string `json:"channel" xml:"showtitle"`
+ Id string `json:"id" xml:"-"`
+ UniqueId UniqueId `json:"-" xml:"uniqueid"`
+ Plot string `json:"description" xml:"plot"`
+ UploadDate string `json:"upload_date" xml:"-"`
+ Aired string `json:"-" xml:"aired"`
+ Season string `json:"-" xml:"season"`
+ Episode string `json:"-" xml:"episode"`
+ Genre string `json:"-" xml:"genre"`
+ InfoPath string `json:"-" xml:"-"`
+}
+
+func LoadEpisode(info_path string) Episode {
+ info, err := os.ReadFile(info_path)
+ if err != nil {
+ panic(err)
+ }
+
+ episode := Episode{}
+
+ err = json.Unmarshal(info, &episode)
+ if err != nil {
+ panic(err)
+ }
+
+ parsed, err := time.Parse("20060102", episode.UploadDate)
+ if err != nil {
+ panic(err)
+ }
+
+ episode.Aired = parsed.String()
+ episode.Season = fmt.Sprintf("s%d", parsed.Year())
+ episode.Episode = fmt.Sprintf("e%02d%02d", parsed.Month(), parsed.Day())
+
+ // these fields cannot be inferred
+ episode.Genre = "YouTube"
+ episode.UniqueId = UniqueId{
+ Default: "true",
+ Type: "youtube",
+ Text: episode.Id,
+ }
+
+ episode.InfoPath = info_path
+
+ return episode
+}
diff --git a/internal/models/show.go b/internal/models/show.go
new file mode 100644
index 0000000..88b9b4e
--- /dev/null
+++ b/internal/models/show.go
@@ -0,0 +1,58 @@
+package models
+
+import (
+ "encoding/json"
+ "encoding/xml"
+ "os"
+)
+
+// Target Show XML structure:
+//
+//
+// #{safe(metadata["title"]}/title>
+// <plot>#{safe(metadata["description"]}</plot>
+// <uniqueid type="youtube" default="true">#{safe(metadata["id"])}</uniqueid>
+// <genre>YouTube</genre>
+// </tvshow>
+
+type Show struct {
+ XMLName xml.Name `xml:"tvshow"`
+ Title string `json:"channel" xml:"title"`
+ Id string `json:"channel_id" xml:"-"`
+ UniqueId UniqueId `json:"-" xml:"uniqueid"`
+ Plot string `json:"description" xml:"plot"`
+ Genre string `json:"-" xml:"genre"`
+ InfoPath string `json:"-" xml:"-"`
+ Thumbnails []Thumbnail `json:"thumbnails" xml:"-"`
+}
+
+type Thumbnail struct {
+ Url string `json:"url" xml:"-"`
+ Id string `json:"id" xml:"-"`
+}
+
+func LoadShow(info_path string) Show {
+ info, err := os.ReadFile(info_path)
+ if err != nil {
+ panic(err)
+ }
+
+ show := Show{}
+
+ err = json.Unmarshal(info, &show)
+ if err != nil {
+ panic(err)
+ }
+
+ // these fields cannot be inferred
+ show.Genre = "YouTube"
+ show.UniqueId = UniqueId{
+ Default: "true",
+ Type: "youtube",
+ Text: show.Id,
+ }
+
+ show.InfoPath = info_path
+
+ return show
+}
diff --git a/internal/models/uniqueid.go b/internal/models/uniqueid.go
new file mode 100644
index 0000000..82f62cd
--- /dev/null
+++ b/internal/models/uniqueid.go
@@ -0,0 +1,7 @@
+package models
+
+type UniqueId struct {
+ Text string `xml:",chardata"`
+ Type string `xml:"type,attr"`
+ Default string `xml:"default,attr"`
+}
diff --git a/internal/nfo/nfo.go b/internal/nfo/nfo.go
new file mode 100644
index 0000000..dbf0c26
--- /dev/null
+++ b/internal/nfo/nfo.go
@@ -0,0 +1,38 @@
+package nfo
+
+import (
+ "encoding/xml"
+ "log"
+ "os"
+ "strings"
+
+ "git.meatbag.se/varl/subsyt/internal/models"
+)
+
+func WriteEpisodeNFO(ep models.Episode, info_path string) {
+ out_path := strings.Replace(info_path, ".info.json", ".nfo", 1)
+
+ log.Printf("writing info from '%s' to '%s'\n", info_path, out_path)
+
+ xmlData, err := xml.MarshalIndent(ep, "", " ")
+ if err != nil {
+ panic(err)
+ }
+
+ complete := xml.Header + string(xmlData)
+ log.Printf("%s", complete)
+ os.WriteFile(out_path, xmlData, 0644)
+}
+
+func WriteShowInfo(show models.Show, out_path string) {
+ log.Printf("writing info from '%s' to '%s'\n", show, out_path)
+
+ xmlData, err := xml.MarshalIndent(show, "", " ")
+ if err != nil {
+ panic(err)
+ }
+
+ complete := xml.Header + string(xmlData)
+ log.Printf("%s", complete)
+ os.WriteFile(out_path, xmlData, 0644)
+}
diff --git a/main.go b/main.go
index 866acd4..18219e7 100644
--- a/main.go
+++ b/main.go
@@ -3,9 +3,11 @@ package main
import (
"log"
"os"
+ "path/filepath"
"git.meatbag.se/varl/subsyt/internal/config"
"git.meatbag.se/varl/subsyt/internal/dl"
+ "git.meatbag.se/varl/subsyt/internal/metadata"
"git.meatbag.se/varl/subsyt/internal/opml"
)
@@ -29,15 +31,16 @@ func main() {
}
for _, outlines := range opml.Body.Outline {
- log.Printf("Archiving videos from OPML: %s", outlines.Title)
+ log.Printf("Archiving videos from OPML: %s\n", outlines.Title)
for _, outline := range outlines.Outlines {
- dl.Get(dl.Download{
+ dl.Youtube(dl.Download{
Url: outline.XmlUrl,
- Name: outline.Title,
- OutDir: cfg.Out_dir,
+ OutDir: filepath.Join(cfg.Out_dir, outline.Title),
DryRun: cfg.Dry_run,
}, provider)
+
+ metadata.Generate(cfg.Out_dir, outline.Title, cfg.Dry_run)
}
}
}