From 09134c46c4d313380e9ce7b67b0e57127ef1e023 Mon Sep 17 00:00:00 2001 From: Viktor Varland Date: Wed, 1 Oct 2025 21:21:59 +0200 Subject: [PATCH] refactor: split logic into distinct parts --- Containerfile | 3 +- README.md | 34 ++++- internal/config/config.go | 19 +-- internal/config/paths.go | 20 +++ internal/fsops/fsops.go | 133 +++++++++++++++++ internal/metadata/art.go | 65 ++++---- internal/metadata/metadata.go | 230 ++++++++++++++++++++--------- internal/metadata/metadata_test.go | 101 +++++++++++++ internal/model/episode.go | 1 + internal/nfo/nfo.go | 9 +- internal/organize/organize.go | 166 +++++++++++++++++++++ internal/organize/organize_test.go | 120 +++++++++++++++ internal/server/server_test.go | 1 + main.go | 67 +++++++-- 14 files changed, 842 insertions(+), 127 deletions(-) create mode 100644 internal/config/paths.go create mode 100644 internal/fsops/fsops.go create mode 100644 internal/metadata/metadata_test.go create mode 100644 internal/organize/organize.go create mode 100644 internal/organize/organize_test.go diff --git a/Containerfile b/Containerfile index b9d4164..247cdd8 100644 --- a/Containerfile +++ b/Containerfile @@ -51,7 +51,8 @@ COPY <<-EOT /data/config.json { "daemon": true, "dry_run": false, - "out_dir": "/data/vids", + "download_dir": "/data/download", + "media_dir": "/data/media", "http_api": { "enable": true, "listen": "0.0.0.0:6901", diff --git a/README.md b/README.md index 7ae8e47..bb83e50 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,12 @@ ## description `subsyt` is a wrapper around `yt-dlp`[1] to download youtube channels -based on a OPML file containing all your subscriptions, sorting the -channels into `{show}/{season}` folders, and generates `nfo` files, -extracts thumbnails, downloads posters, banners, and fanart so the media -should plug into media libraries well-enough, e.g. Jellyfin and Kodi. +based on a OPML file containing all your subscriptions. Downloads land in +an isolated staging directory before being organised into +`{show}/{season}` folders under your configured media library. During the +organising step the tool generates `nfo` files, extracts thumbnails, +downloads posters, banners, and fanart so the media should plug into +media libraries well-enough, e.g. Jellyfin and Kodi. A quick rundown on how to use it: @@ -73,7 +75,8 @@ Full `config.json`: { "daemon": true, "dry_run": true, - "out_dir": "./vids", + "download_dir": "./vids/_staging", + "media_dir": "./vids", "provider": { "youtube": { "verbose": false, @@ -97,7 +100,8 @@ Minimal `config.json`: ```json { - "out_dir": "./vids", + "download_dir": "./vids/_staging", + "media_dir": "./vids", "provider": { "youtube": { "cmd": "./yt-dlp", @@ -108,6 +112,24 @@ Minimal `config.json`: } ``` +## migration + +Existing deployments that used to read media directly from +`download_dir` should be migrated manually: + +1. Stop the daemon or API workers so new downloads pause. +2. Back up your current `media_dir` and staging tree. +3. Move the contents of the legacy `download_dir/shows` and + `download_dir/episodes` directories into the new `media_dir`, sorted + by show and season as desired. +4. Optionally re-run `subsyt` with `dry_run=true` to confirm the new + layout before enabling writes again. +5. Once satisfied, delete the obsolete per-show/per-episode staging + directories under `download_dir`. + +The application now keeps raw downloads inside `download_dir` and writes +the organised library exclusively to `media_dir`. + ## generate opml Use this javascript snippet: diff --git a/internal/config/config.go b/internal/config/config.go index 5929693..500b607 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,19 +22,20 @@ type Provider struct { } type Http_api struct { - Enable bool - Listen string - Auth_token string + Enable bool + Listen string + Auth_token string Auth_token_file string - Queue_file string + Queue_file string } type Config struct { - Out_dir string - Provider map[string]Provider - Dry_run bool - Daemon bool - Http_api Http_api + Download_dir string + Media_dir string + Provider map[string]Provider + Dry_run bool + Daemon bool + Http_api Http_api } func Load(filepath string) (Config, error) { diff --git a/internal/config/paths.go b/internal/config/paths.go new file mode 100644 index 0000000..9f6f046 --- /dev/null +++ b/internal/config/paths.go @@ -0,0 +1,20 @@ +package config + +import "path/filepath" + +type Paths struct { + DownloadRoot string + ChannelsDir string + EpisodesDir string + MediaDir string +} + +func (c Config) Paths() Paths { + root := c.Download_dir + return Paths{ + DownloadRoot: root, + ChannelsDir: filepath.Join(root, "channels"), + EpisodesDir: filepath.Join(root, "episodes"), + MediaDir: c.Media_dir, + } +} diff --git a/internal/fsops/fsops.go b/internal/fsops/fsops.go new file mode 100644 index 0000000..02366d0 --- /dev/null +++ b/internal/fsops/fsops.go @@ -0,0 +1,133 @@ +package fsops + +import ( + "errors" + "fmt" + "log" + "os" + "path/filepath" +) + +type Manager struct { + DryRun bool +} + +func (m Manager) EnsureDir(path string) error { + if path == "" { + return fmt.Errorf("ensure dir: empty path") + } + + if m.DryRun { + log.Printf("[dry-run] ensure dir %s\n", path) + return nil + } + + if err := os.MkdirAll(path, 0o755); err != nil { + return fmt.Errorf("ensure dir %s: %w", path, err) + } + return nil +} + +func (m Manager) Move(src, dst string) error { + if src == "" || dst == "" { + return fmt.Errorf("move: empty source or destination") + } + + dstDir := filepath.Dir(dst) + if err := m.EnsureDir(dstDir); err != nil { + return err + } + + if m.DryRun { + log.Printf("[dry-run] move %s -> %s\n", src, dst) + return nil + } + + if err := os.Rename(src, dst); err != nil { + return fmt.Errorf("move %s -> %s: %w", src, dst, err) + } + return nil +} + +func (m Manager) Remove(path string) error { + if path == "" { + return fmt.Errorf("remove: empty path") + } + + if m.DryRun { + log.Printf("[dry-run] remove %s\n", path) + return nil + } + + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove %s: %w", path, err) + } + return nil +} + +func (m Manager) RemoveAll(path string) error { + if path == "" { + return fmt.Errorf("remove all: empty path") + } + + if m.DryRun { + log.Printf("[dry-run] remove tree %s\n", path) + return nil + } + + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("remove all %s: %w", path, err) + } + return nil +} + +func (m Manager) RemoveEmptyDirs(root string) error { + if root == "" { + return fmt.Errorf("remove empty dirs: empty root") + } + + if _, err := os.Stat(root); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + + if path == root { + return nil + } + + if !d.IsDir() { + return nil + } + + entries, readErr := os.ReadDir(path) + if readErr != nil { + if errors.Is(readErr, os.ErrNotExist) { + return nil + } + return readErr + } + + if len(entries) > 0 { + return nil + } + + if m.DryRun { + log.Printf("[dry-run] remove empty dir %s\n", path) + return nil + } + + if rmErr := os.Remove(path); rmErr != nil && !os.IsNotExist(rmErr) { + return fmt.Errorf("remove empty dir %s: %w", path, rmErr) + } + return nil + }) +} diff --git a/internal/metadata/art.go b/internal/metadata/art.go index 00532e3..c26ddac 100644 --- a/internal/metadata/art.go +++ b/internal/metadata/art.go @@ -10,32 +10,30 @@ import ( "git.meatbag.se/varl/subsyt/internal/model" ) -func episodeImage(path string) { - if strings.Contains(path, "-thumb") { - log.Printf("thumbnail detected '%s'\n", path) +func NormalizeEpisodeThumbnail(name string) string { + lower := strings.ToLower(name) + if strings.Contains(lower, "-thumb") { + return name + } + + ext := filepath.Ext(name) + switch strings.ToLower(ext) { + case ".jpg", ".jpeg": + base := strings.TrimSuffix(name, ext) + return base + "-thumb" + ext + } + return name +} + +func EnsureBanner(show model.Show, destDir string, dryRun bool) { + bannerPath := filepath.Join(destDir, "banner.jpg") + if _, err := os.Stat(bannerPath); err == nil { + log.Printf("%s has a banner, skipping download\n", show.Title) return } - thumb := strings.Replace(path, ".jpg", "-thumb.jpg", 1) - log.Printf("renaming thumbnail from '%s' to '%s'\n", path, thumb) - err := os.Rename(path, thumb) - if err != nil { - log.Printf("failed to rename '%s' to '%s\n'", path, thumb) - } -} -func showPoster(path string, show_dir string) { - poster := filepath.Join(show_dir, "poster.jpg") - log.Printf("renaming show image from '%s' to '%s'\n", path, poster) - err := os.Rename(path, poster) - if err != nil { - log.Printf("failed to rename '%s' to '%s\n'", path, poster) - } -} - -func showBanner(show model.Show, showDir string) { - _, err := os.Stat(filepath.Join(showDir, "banner.jpg")) - if err == nil { - log.Printf("%s has a banner, skipping download\n", show.Title) + if dryRun { + log.Printf("[dry-run] would download banner for %s\n", show.Title) return } @@ -45,20 +43,26 @@ func showBanner(show model.Show, showDir string) { log.Println("found banner candidate") dl.Fetch(dl.Download{ Url: thumb.Url, - OutDir: showDir, + OutDir: destDir, Name: "banner.jpg", }) + return } } } -func showFanart(show model.Show, showDir string) { - _, err := os.Stat(filepath.Join(showDir, "fanart.jpg")) - if err == nil { +func EnsureFanart(show model.Show, destDir string, dryRun bool) { + fanartPath := filepath.Join(destDir, "fanart.jpg") + if _, err := os.Stat(fanartPath); err == nil { log.Printf("%s has fanart, skipping download\n", show.Title) return } + if dryRun { + log.Printf("[dry-run] would download fanart for %s\n", show.Title) + return + } + c := model.Thumbnail{} for index, thumb := range show.Thumbnails { log.Println(index, thumb) @@ -68,9 +72,14 @@ func showFanart(show model.Show, showDir string) { } } + if c.Url == "" { + log.Printf("no fanart candidate for %s\n", show.Title) + return + } + dl.Fetch(dl.Download{ Url: c.Url, - OutDir: showDir, + OutDir: destDir, Name: "fanart.jpg", }) } diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go index a179281..0e30c70 100644 --- a/internal/metadata/metadata.go +++ b/internal/metadata/metadata.go @@ -1,92 +1,188 @@ package metadata import ( + "errors" "fmt" "log" "os" "path/filepath" - "regexp" "strings" "git.meatbag.se/varl/subsyt/internal/model" - "git.meatbag.se/varl/subsyt/internal/nfo" ) -func findFiles(scanPath string, ext string) ([]string, error) { - var result []string +type ShowAsset struct { + Show model.Show + InfoPath string + Dir string + Images []string +} - err := filepath.Walk(scanPath, func(path string, info os.FileInfo, err error) error { +type EpisodeAsset struct { + Episode model.Episode + InfoPath string + Dir string + MediaPath string + Sidecars []string +} + +func findFiles(root, ext string) ([]string, error) { + if root == "" { + return nil, fmt.Errorf("find files: empty root") + } + + if _, err := os.Stat(root); errors.Is(err, os.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("find files: %w", err) + } + + var result []string + walkErr := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return err } - if !info.IsDir() && strings.HasSuffix(path, ext) { + if d.IsDir() { + return nil + } + if strings.HasSuffix(path, ext) { result = append(result, path) } return nil }) - - if err != nil { - return nil, fmt.Errorf("error walking directory: %w", err) + if walkErr != nil { + return nil, fmt.Errorf("walk %s: %w", root, walkErr) } - return result, nil } -func Generate(outDir string, title string, dryRun bool) { - showDir := filepath.Join(outDir, title) - log.Printf("Writing NFO's for %s\n", showDir) - - if dryRun { - return - } - - infojsons, err := findFiles(showDir, ".info.json") - if err != nil { - panic(err) - } - - show := regexp.MustCompile("s(NA)") - season := regexp.MustCompile(`s\d\d\d\d`) - - for index, path := range infojsons { - log.Println(index, path) - switch { - case show.MatchString(path): - show := model.LoadShow(path) - nfo.WriteShowInfo(show, filepath.Join(showDir, "tvshow.nfo")) - showBanner(show, showDir) - showFanart(show, showDir) - case season.MatchString(path): - ep := model.LoadEpisode(path) - nfo.WriteEpisodeNFO(ep, path) - default: - log.Printf("no match for '%s'\n", path) - } - - os.Remove(path) - } - - images, err := findFiles(showDir, ".jpg") - if err != nil { - panic(err) - } - - for index, path := range images { - log.Println(index, path) - switch { - case show.MatchString(path): - showPoster(path, showDir) - case season.MatchString(path): - episodeImage(path) - default: - log.Printf("no match for '%s'\n", path) - } - } - - del := filepath.Join(showDir, "sNA") - log.Printf("removing '%s'\n", del) - err = os.RemoveAll(del) - if err != nil { - log.Println("failed to remove", err) - } +func trimInfoSuffix(path string) string { + return strings.TrimSuffix(path, ".info.json") +} + +func globSidecars(base string) ([]string, error) { + matches, err := filepath.Glob(base + ".*") + if err != nil { + return nil, fmt.Errorf("glob %s.*: %w", base, err) + } + + var sidecars []string + for _, match := range matches { + if strings.HasSuffix(match, ".info.json") { + continue + } + sidecars = append(sidecars, match) + } + return sidecars, nil +} + +func directoryImages(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("readdir %s: %w", dir, err) + } + + var images []string + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + switch strings.ToLower(filepath.Ext(name)) { + case ".jpg", ".jpeg", ".png", ".webp": + images = append(images, filepath.Join(dir, name)) + } + } + return images, nil +} + +func resolveMediaPath(base string, episode model.Episode, sidecars []string) (string, []string) { + if episode.Ext != "" { + candidate := base + "." + episode.Ext + if _, err := os.Stat(candidate); err == nil { + var rest []string + for _, path := range sidecars { + if path != candidate { + rest = append(rest, path) + } + } + return candidate, rest + } + } + + for _, path := range sidecars { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".mp4", ".mkv", ".webm", ".m4v", ".mov", ".avi": + var rest []string + for _, other := range sidecars { + if other != path { + rest = append(rest, other) + } + } + return path, rest + } + } + + return "", sidecars +} + +func ScanShows(root string) ([]ShowAsset, error) { + infoFiles, err := findFiles(root, ".info.json") + if err != nil { + return nil, err + } + + var assets []ShowAsset + for _, infoPath := range infoFiles { + show := model.LoadShow(infoPath) + dir := filepath.Dir(infoPath) + + imgs, imgErr := directoryImages(dir) + if imgErr != nil { + log.Printf("scan shows: %v\n", imgErr) + } + + assets = append(assets, ShowAsset{ + Show: show, + InfoPath: infoPath, + Dir: dir, + Images: imgs, + }) + } + + return assets, nil +} + +func ScanEpisodes(root string) ([]EpisodeAsset, error) { + infoFiles, err := findFiles(root, ".info.json") + if err != nil { + return nil, err + } + + var assets []EpisodeAsset + for _, infoPath := range infoFiles { + episode := model.LoadEpisode(infoPath) + base := trimInfoSuffix(infoPath) + dir := filepath.Dir(infoPath) + + sidecars, sidecarErr := globSidecars(base) + if sidecarErr != nil { + return nil, sidecarErr + } + + mediaPath, remaining := resolveMediaPath(base, episode, sidecars) + assets = append(assets, EpisodeAsset{ + Episode: episode, + InfoPath: infoPath, + Dir: dir, + MediaPath: mediaPath, + Sidecars: remaining, + }) + } + + return assets, nil } diff --git a/internal/metadata/metadata_test.go b/internal/metadata/metadata_test.go new file mode 100644 index 0000000..aaae497 --- /dev/null +++ b/internal/metadata/metadata_test.go @@ -0,0 +1,101 @@ +package metadata + +import ( + "os" + "path/filepath" + "testing" +) + +func TestScanEpisodes(t *testing.T) { + root := t.TempDir() + seasonDir := filepath.Join(root, "s2024") + if err := os.MkdirAll(seasonDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + base := filepath.Join(seasonDir, "TestChannel.s2024Se0210S.Sample.abc123") + infoPath := base + ".info.json" + videoPath := base + ".mp4" + thumbPath := base + ".jpg" + + epJSON := `{"title":"Episode","channel":"Test Channel","id":"abc123","description":"Episode description","upload_date":"20240210","ext":"mp4"}` + if err := os.WriteFile(infoPath, []byte(epJSON), 0o644); err != nil { + t.Fatalf("write info: %v", err) + } + if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil { + t.Fatalf("write video: %v", err) + } + if err := os.WriteFile(thumbPath, []byte("thumb"), 0o644); err != nil { + t.Fatalf("write thumbnail: %v", err) + } + + assets, err := ScanEpisodes(root) + if err != nil { + t.Fatalf("ScanEpisodes error: %v", err) + } + if len(assets) != 1 { + t.Fatalf("expected 1 asset, got %d", len(assets)) + } + + asset := assets[0] + if asset.MediaPath != videoPath { + t.Fatalf("unexpected media path: %s", asset.MediaPath) + } + if len(asset.Sidecars) != 1 || asset.Sidecars[0] != thumbPath { + t.Fatalf("unexpected sidecars: %#v", asset.Sidecars) + } +} + +func TestScanShows(t *testing.T) { + root := t.TempDir() + showDir := filepath.Join(root, "sNA") + if err := os.MkdirAll(showDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + base := filepath.Join(showDir, "test-channel") + infoPath := base + ".info.json" + posterPath := base + ".jpg" + + showJSON := `{"channel":"Test Channel","channel_id":"chan123","description":"Show description","thumbnails":[]}` + if err := os.WriteFile(infoPath, []byte(showJSON), 0o644); err != nil { + t.Fatalf("write show info: %v", err) + } + if err := os.WriteFile(posterPath, []byte("poster"), 0o644); err != nil { + t.Fatalf("write poster: %v", err) + } + + assets, err := ScanShows(root) + if err != nil { + t.Fatalf("ScanShows error: %v", err) + } + if len(assets) != 1 { + t.Fatalf("expected 1 show asset, got %d", len(assets)) + } + + asset := assets[0] + if asset.InfoPath != infoPath { + t.Fatalf("unexpected info path: %s", asset.InfoPath) + } + if len(asset.Images) != 1 || asset.Images[0] != posterPath { + t.Fatalf("unexpected images: %#v", asset.Images) + } +} + +func TestNormalizeEpisodeThumbnail(t *testing.T) { + got := NormalizeEpisodeThumbnail("file.jpg") + expected := "file-thumb.jpg" + if got != expected { + t.Fatalf("expected %s, got %s", expected, got) + } + + got = NormalizeEpisodeThumbnail("file-thumb.jpg") + if got != "file-thumb.jpg" { + t.Fatalf("normalization should keep thumb suffix, got %s", got) + } + + got = NormalizeEpisodeThumbnail("file.jpeg") + if got != "file-thumb.jpeg" { + t.Fatalf("unexpected jpeg normalization: %s", got) + } +} diff --git a/internal/model/episode.go b/internal/model/episode.go index c48f067..9f0127d 100644 --- a/internal/model/episode.go +++ b/internal/model/episode.go @@ -29,6 +29,7 @@ type Episode struct { Id string `json:"id" xml:"-"` UniqueId UniqueId `json:"-" xml:"uniqueid"` Plot string `json:"description" xml:"plot"` + Ext string `json:"ext" xml:"-"` UploadDate string `json:"upload_date" xml:"-"` Aired string `json:"-" xml:"aired"` Season string `json:"-" xml:"season"` diff --git a/internal/nfo/nfo.go b/internal/nfo/nfo.go index b35be87..72b2d13 100644 --- a/internal/nfo/nfo.go +++ b/internal/nfo/nfo.go @@ -4,15 +4,12 @@ import ( "encoding/xml" "log" "os" - "strings" "git.meatbag.se/varl/subsyt/internal/model" ) -func WriteEpisodeNFO(ep model.Episode, info_path string) { - out_path := strings.Replace(info_path, ".info.json", ".nfo", 1) - - log.Printf("writing info from '%s' to '%s'\n", info_path, out_path) +func WriteEpisodeNFO(ep model.Episode, outPath string) { + log.Printf("writing episode nfo to '%s'\n", outPath) xmlData, err := xml.MarshalIndent(ep, "", " ") if err != nil { @@ -21,7 +18,7 @@ func WriteEpisodeNFO(ep model.Episode, info_path string) { complete := xml.Header + string(xmlData) log.Printf("%s", complete) - os.WriteFile(out_path, xmlData, 0644) + os.WriteFile(outPath, xmlData, 0644) } func WriteShowInfo(show model.Show, out_path string) { diff --git a/internal/organize/organize.go b/internal/organize/organize.go new file mode 100644 index 0000000..fbc2161 --- /dev/null +++ b/internal/organize/organize.go @@ -0,0 +1,166 @@ +package organize + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "git.meatbag.se/varl/subsyt/internal/config" + "git.meatbag.se/varl/subsyt/internal/fsops" + "git.meatbag.se/varl/subsyt/internal/metadata" + "git.meatbag.se/varl/subsyt/internal/nfo" +) + +type Organizer struct { + Paths config.Paths + FS fsops.Manager +} + +func (o Organizer) Prepare() error { + if err := o.FS.EnsureDir(o.Paths.DownloadRoot); err != nil { + return err + } + if err := o.FS.EnsureDir(o.Paths.ChannelsDir); err != nil { + return err + } + if err := o.FS.EnsureDir(o.Paths.EpisodesDir); err != nil { + return err + } + if err := o.FS.EnsureDir(o.Paths.MediaDir); err != nil { + return err + } + return nil +} + +func (o Organizer) ProcessShows(shows []metadata.ShowAsset) error { + for _, show := range shows { + showDir := filepath.Join(o.Paths.MediaDir, safeName(show.Show.Title)) + if err := o.FS.EnsureDir(showDir); err != nil { + return err + } + + if o.FS.DryRun { + log.Printf("[dry-run] write tvshow.nfo for %s\n", show.Show.Title) + } else { + nfo.WriteShowInfo(show.Show, filepath.Join(showDir, "tvshow.nfo")) + } + + for _, img := range show.Images { + name := categorizeShowImage(img) + if name == "" { + continue + } + dest := filepath.Join(showDir, name) + if _, err := os.Stat(dest); err == nil { + log.Printf("skip image move, destination exists: %s\n", dest) + if err := o.FS.Remove(img); err != nil { + return err + } + continue + } + if err := o.FS.Move(img, dest); err != nil { + return err + } + } + + metadata.EnsureBanner(show.Show, showDir, o.FS.DryRun) + metadata.EnsureFanart(show.Show, showDir, o.FS.DryRun) + + if err := o.FS.Remove(show.InfoPath); err != nil { + return err + } + } + + if err := o.FS.RemoveEmptyDirs(o.Paths.ChannelsDir); err != nil { + return err + } + return nil +} + +func (o Organizer) ProcessEpisodes(episodes []metadata.EpisodeAsset) error { + for _, ep := range episodes { + showDir := filepath.Join(o.Paths.MediaDir, safeName(ep.Episode.ShowTitle)) + seasonDir := filepath.Join(showDir, ep.Episode.Season) + + if err := o.FS.EnsureDir(seasonDir); err != nil { + return err + } + + var destMediaPath string + if ep.MediaPath != "" { + filename := filepath.Base(ep.MediaPath) + destMediaPath = filepath.Join(seasonDir, filename) + if err := o.FS.Move(ep.MediaPath, destMediaPath); err != nil { + return err + } + } else { + log.Printf("no media file for %s (%s)\n", ep.Episode.Title, ep.InfoPath) + } + + for _, sidecar := range ep.Sidecars { + filename := filepath.Base(sidecar) + ext := strings.ToLower(filepath.Ext(filename)) + if ext == ".jpg" || ext == ".jpeg" { + filename = metadata.NormalizeEpisodeThumbnail(filename) + } + dest := filepath.Join(seasonDir, filename) + if _, err := os.Stat(dest); err == nil { + log.Printf("skip sidecar move, destination exists: %s\n", dest) + if err := o.FS.Remove(sidecar); err != nil { + return err + } + continue + } + if err := o.FS.Move(sidecar, dest); err != nil { + return err + } + } + + if destMediaPath != "" { + base := strings.TrimSuffix(filepath.Base(destMediaPath), filepath.Ext(destMediaPath)) + nfoPath := filepath.Join(seasonDir, fmt.Sprintf("%s.nfo", base)) + if o.FS.DryRun { + log.Printf("[dry-run] write episode nfo %s\n", nfoPath) + } else { + nfo.WriteEpisodeNFO(ep.Episode, nfoPath) + } + } + + if err := o.FS.Remove(ep.InfoPath); err != nil { + return err + } + } + + if err := o.FS.RemoveEmptyDirs(o.Paths.EpisodesDir); err != nil { + return err + } + return nil +} + +func safeName(name string) string { + replacer := strings.NewReplacer("/", "-", "\\", "-", ":", "-", "?", "", "*", "", "\"", "", "<", "", ">", "", "|", "") + sanitized := replacer.Replace(strings.TrimSpace(name)) + if sanitized == "" { + return "unknown" + } + return sanitized +} + +func categorizeShowImage(path string) string { + lower := strings.ToLower(filepath.Base(path)) + switch { + case strings.Contains(lower, "banner"): + return "banner.jpg" + case strings.Contains(lower, "fanart"): + return "fanart.jpg" + case strings.Contains(lower, "poster"): + return "poster.jpg" + default: + if strings.HasSuffix(lower, ".jpg") || strings.HasSuffix(lower, ".jpeg") { + return "poster.jpg" + } + } + return "" +} diff --git a/internal/organize/organize_test.go b/internal/organize/organize_test.go new file mode 100644 index 0000000..dc5094f --- /dev/null +++ b/internal/organize/organize_test.go @@ -0,0 +1,120 @@ +package organize + +import ( + "os" + "path/filepath" + "testing" + + "git.meatbag.se/varl/subsyt/internal/config" + "git.meatbag.se/varl/subsyt/internal/fsops" + "git.meatbag.se/varl/subsyt/internal/metadata" +) + +func TestOrganizerProcess(t *testing.T) { + root := t.TempDir() + + paths := config.Paths{ + DownloadRoot: filepath.Join(root, "downloads"), + ChannelsDir: filepath.Join(root, "downloads", "channels"), + EpisodesDir: filepath.Join(root, "downloads", "episodes"), + MediaDir: filepath.Join(root, "library"), + } + + if err := os.MkdirAll(paths.ChannelsDir, 0o755); err != nil { + t.Fatalf("mkdir channels: %v", err) + } + if err := os.MkdirAll(paths.EpisodesDir, 0o755); err != nil { + t.Fatalf("mkdir episodes: %v", err) + } + + showSub := filepath.Join(paths.ChannelsDir, "sNA") + if err := os.MkdirAll(showSub, 0o755); err != nil { + t.Fatalf("mkdir show subdir: %v", err) + } + + showBase := filepath.Join(showSub, "test-channel") + showInfo := showBase + ".info.json" + showImage := showBase + ".jpg" + showJSON := `{"channel":"Test Channel","channel_id":"chan123","description":"Show description","thumbnails":[]}` + if err := os.WriteFile(showInfo, []byte(showJSON), 0o644); err != nil { + t.Fatalf("write show info: %v", err) + } + if err := os.WriteFile(showImage, []byte("poster"), 0o644); err != nil { + t.Fatalf("write show image: %v", err) + } + + seasonDir := filepath.Join(paths.EpisodesDir, "s2024") + if err := os.MkdirAll(seasonDir, 0o755); err != nil { + t.Fatalf("mkdir season: %v", err) + } + base := filepath.Join(seasonDir, "Test Channel.s2024Se0210S.Sample.abc123") + infoPath := base + ".info.json" + videoPath := base + ".mp4" + subsPath := base + ".en.vtt" + thumbPath := base + ".jpg" + epJSON := `{"title":"Episode","channel":"Test Channel","id":"abc123","description":"Episode description","upload_date":"20240210","ext":"mp4"}` + if err := os.WriteFile(infoPath, []byte(epJSON), 0o644); err != nil { + t.Fatalf("write episode info: %v", err) + } + if err := os.WriteFile(videoPath, []byte("video"), 0o644); err != nil { + t.Fatalf("write episode video: %v", err) + } + if err := os.WriteFile(subsPath, []byte("subtitles"), 0o644); err != nil { + t.Fatalf("write subtitles: %v", err) + } + if err := os.WriteFile(thumbPath, []byte("thumb"), 0o644); err != nil { + t.Fatalf("write thumb: %v", err) + } + + shows, err := metadata.ScanShows(paths.ChannelsDir) + if err != nil { + t.Fatalf("scan shows: %v", err) + } + episodes, err := metadata.ScanEpisodes(paths.EpisodesDir) + if err != nil { + t.Fatalf("scan episodes: %v", err) + } + + organizer := Organizer{ + Paths: paths, + FS: fsops.Manager{DryRun: false}, + } + if err := organizer.Prepare(); err != nil { + t.Fatalf("prepare: %v", err) + } + + if err := organizer.ProcessShows(shows); err != nil { + t.Fatalf("process shows: %v", err) + } + if err := organizer.ProcessEpisodes(episodes); err != nil { + t.Fatalf("process episodes: %v", err) + } + + showDir := filepath.Join(paths.MediaDir, "Test Channel") + seasonOut := filepath.Join(showDir, "s2024") + if _, err := os.Stat(filepath.Join(showDir, "tvshow.nfo")); err != nil { + t.Fatalf("expected tvshow.nfo: %v", err) + } + if _, err := os.Stat(filepath.Join(showDir, "poster.jpg")); err != nil { + t.Fatalf("expected poster.jpg: %v", err) + } + if _, err := os.Stat(filepath.Join(seasonOut, filepath.Base(videoPath))); err != nil { + t.Fatalf("expected moved video: %v", err) + } + if _, err := os.Stat(filepath.Join(seasonOut, "Test Channel.s2024Se0210S.Sample.abc123-thumb.jpg")); err != nil { + t.Fatalf("expected moved thumbnail: %v", err) + } + if _, err := os.Stat(filepath.Join(seasonOut, "Test Channel.s2024Se0210S.Sample.abc123.en.vtt")); err != nil { + t.Fatalf("expected moved subtitles: %v", err) + } + if _, err := os.Stat(filepath.Join(seasonOut, "Test Channel.s2024Se0210S.Sample.abc123.nfo")); err != nil { + t.Fatalf("expected episode nfo: %v", err) + } + + if _, err := os.Stat(infoPath); !os.IsNotExist(err) { + t.Fatalf("expected episode info removed, got err=%v", err) + } + if _, err := os.Stat(showInfo); !os.IsNotExist(err) { + t.Fatalf("expected show info removed, got err=%v", err) + } +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index e38e280..ce3fd25 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "path/filepath" "testing" "time" diff --git a/main.go b/main.go index 9fd67ad..6c017a3 100644 --- a/main.go +++ b/main.go @@ -7,18 +7,29 @@ import ( "fmt" "log" "os" - "path/filepath" "time" "git.meatbag.se/varl/subsyt/internal/config" "git.meatbag.se/varl/subsyt/internal/dl" "git.meatbag.se/varl/subsyt/internal/format" + "git.meatbag.se/varl/subsyt/internal/fsops" "git.meatbag.se/varl/subsyt/internal/metadata" + "git.meatbag.se/varl/subsyt/internal/organize" "git.meatbag.se/varl/subsyt/internal/scheduler" "git.meatbag.se/varl/subsyt/internal/server" ) func run(cfg config.Config) { + paths := cfg.Paths() + organizer := organize.Organizer{ + Paths: paths, + FS: fsops.Manager{DryRun: cfg.Dry_run}, + } + + if err := organizer.Prepare(); err != nil { + log.Fatalf("prepare directories: %v", err) + } + provider := cfg.Provider["youtube"] if err := dl.UpgradeYtDlp(provider.Cmd); err != nil { @@ -48,7 +59,7 @@ func run(cfg config.Config) { dl.Youtube(dl.Download{ Url: feed.Author.Uri, - OutDir: filepath.Join(cfg.Out_dir, outline.Title), + OutDir: paths.ChannelsDir, DryRun: cfg.Dry_run, Metadata: true, }, provider) @@ -61,15 +72,39 @@ func run(cfg config.Config) { log.Printf("Entry: %#v", entry) dl.Youtube(dl.Download{ Url: url, - OutDir: filepath.Join(cfg.Out_dir, feed.Title), + OutDir: paths.EpisodesDir, DryRun: cfg.Dry_run, Metadata: false, }, provider) } - - metadata.Generate(cfg.Out_dir, feed.Title, cfg.Dry_run) } } + + if err := syncLibrary(organizer); err != nil { + log.Fatalf("organize library: %v", err) + } +} + +func syncLibrary(organizer organize.Organizer) error { + shows, err := metadata.ScanShows(organizer.Paths.ChannelsDir) + if err != nil { + return fmt.Errorf("scan shows: %w", err) + } + + if err := organizer.ProcessShows(shows); err != nil { + return fmt.Errorf("process shows: %w", err) + } + + episodes, err := metadata.ScanEpisodes(organizer.Paths.EpisodesDir) + if err != nil { + return fmt.Errorf("scan episodes: %w", err) + } + + if err := organizer.ProcessEpisodes(episodes); err != nil { + return fmt.Errorf("process episodes: %w", err) + } + + return nil } func main() { @@ -119,6 +154,8 @@ func main() { } func setupAPIServer(ctx context.Context, cfg config.Config, provider config.Provider) error { + paths := cfg.Paths() + queue, err := server.NewQueue(cfg.Http_api.Queue_file) if err != nil { return fmt.Errorf("load queue: %w", err) @@ -126,7 +163,7 @@ func setupAPIServer(ctx context.Context, cfg config.Config, provider config.Prov go queueWorker(ctx, queue, cfg, provider) - srv := server.NewServer(cfg.Http_api, cfg.Out_dir, queue) + srv := server.NewServer(cfg.Http_api, paths.MediaDir, queue) go func() { if err := srv.Start(ctx); err != nil { log.Printf("http api server stopped with error: %v", err) @@ -137,6 +174,16 @@ func setupAPIServer(ctx context.Context, cfg config.Config, provider config.Prov } func queueWorker(ctx context.Context, q *server.Queue, cfg config.Config, provider config.Provider) { + organizer := organize.Organizer{ + Paths: cfg.Paths(), + FS: fsops.Manager{DryRun: cfg.Dry_run}, + } + + if err := organizer.Prepare(); err != nil { + log.Printf("queue prepare error: %v", err) + return + } + for { item, err := q.Next(ctx) if err != nil { @@ -152,16 +199,16 @@ func queueWorker(ctx context.Context, q *server.Queue, cfg config.Config, provid continue } - outPath := filepath.Join(cfg.Out_dir, "_misc") - dl.Youtube(dl.Download{ Url: item.Request.URL, - OutDir: outPath, + OutDir: organizer.Paths.EpisodesDir, DryRun: cfg.Dry_run, Metadata: false, }, provider) - metadata.Generate(cfg.Out_dir, "_misc", cfg.Dry_run) + if err := syncLibrary(organizer); err != nil { + log.Printf("queue organize error: %v", err) + } if err := q.MarkDone(item.ID); err != nil { log.Printf("queue mark done failed: %v", err)