DL updates

yt-dlp/Download-RegulationGameplay_NEW.ps1

@@ -3,7 +3,6 @@
 $URL = "https://www.youtube.com/@TheRegulationPodcast"
 $folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$'
 
-
 $rejects = @()
 $RejectString = $rejects -join '|'
 

yt-dlp/Download-RegulationPatreon_NEW.ps1

@@ -1,9 +1,8 @@
-## YT Regulation Podcast
+# YT Regulation Patreon
 # Set URL and Location
-$URL = "https://www.youtube.com/@theregulationpod"
+$URL = "https://www.patreon.com/cw/TheRegulationPod"
 $folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$'
 
-
 $rejects = @()
 $RejectString = $rejects -join '|'
 
@@ -26,18 +25,20 @@ function Download-YTDLP {
     
     # Start building an array of arguments using strict comma separation (Recommended format)
     $arguments = @(
-        $videoUrl,
+        "$videoUrl",
         "--write-info-json",
         "--write-thumbnail",
+        "--embed-metadata",
         "--convert-thumbnails", "jpg",
         "--no-progress", 
         "-t", "sleep",
         "--rate-limit", "5M", 
-        "-o", $OutputPath,
+        "-o", "$OutputPath",
         '-f', "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
         "-S", "vcodec:h264",
-        "--reject-title", $RejectString,
+        "--reject-title", "$RejectString",
-        "--playlist-reverse" 
+        "--playlist-reverse",
+        "--download-archive", "ytdlp_archive.txt"
     )
 
     # --- Conditional Logic Applied Here ---
@@ -66,10 +67,10 @@ function Download-YTDLP {
 # Function to manage Short ID tracking (NEW)
 function Manage-ShortTracking {
     Param(
-        [Parameter(Mandatory=$true)]
+        [Parameter(Mandatory = $true)]
         [string]$ShortDirPath,
 
-        [Parameter(Mandatory=$true)]
+        [Parameter(Mandatory = $true)]
         [string]$VideoId,
 
         [switch]$IsNewDownload
@@ -85,8 +86,9 @@ function Manage-ShortTracking {
                 if ($existingIds -contains $VideoId) {
                     return $true # Video ID found, should skip download
                 }
-            } catch {
+            }
-                 Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
+            catch {
+                Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
             }
         }
 
@@ -99,48 +101,87 @@ function Manage-ShortTracking {
                     $newIds = $existingIds + $VideoId
                     $newIds | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile
                     Write-Verbose "Successfully added ID '$VideoId' to shorts tracking."
-                } else {
+                }
-                     Write-Verbose "ID '$VideoId' already exists in tracking file."
+                else {
+                    Write-Verbose "ID '$VideoId' already exists in tracking file."
                 }
 
-            } else {
+            }
-                 # File does not exist, create it
+            else {
+                # File does not exist, create it
                 @($VideoId) | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile
                 Write-Verbose "Created and added ID '$VideoId' to shorts tracking file."
             }
-        } catch {
+        }
+        catch {
             Write-Error "Failed to write to shorts tracking file: $($_.Exception.Message)"
         }
-    } else { 
+    }
+    else { 
         # Check mode (only needed when checking before download)
-         if (Test-Path $trackingFile) {
+        if (Test-Path $trackingFile) {
             try {
                 $existingIds = Get-Content -Path $trackingFile | ConvertFrom-Json
                 return $existingIds -contains $VideoId # Returns True if ID is found
-            } catch {
+            }
-                 Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
+            catch {
-                 return $false # Assume safe to download if parsing fails
+                Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
+                return $false # Assume safe to download if parsing fails
             }
         }
         return $false # No tracking file exists, assume safe to download
     }
 }
 
+function Parse-URLToIdentifier {
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Url
+    )
+
+    $platform = ""
+    $name = ""
+
+    # 1. Determine the Platform
+    if ($Url -match "patreon\.com") {
+        $platform = "patreon"
+        # For Patreon, take the last segment of the URL path (e.g., TheRegulationPod)
+        $name = ($Url -split '/')[-1]
+    }
+    elseif ($Url -match "youtube.com") {
+        $platform = "youtube"
+        # For YouTube, extract everything after the "@" symbol
+        if ($Url -match '@(.*)') {
+            $name = $Matches[1]
+        }
+        else {
+            # Handle cases where @ might not be present (though unlikely for channel URLs)
+            $name = "UnknownChannel" 
+        }
+    }
+    else {
+        Write-Error "Unsupported URL platform."
+        return $null
+    }
+
+    # Construct and return the final identifier
+    return "${platform}_${name}"
+}
+
+$archive_name = Parse-URLToIdentifier -Url $Url
 
 # Download flat playlist
 Write-Verbose -Message "Download flat playlist"
 $flat = D:\yt-dlp\yt-dlp.exe -j --flat-playlist $URL
-$flat | Set-Content -Path YT_theregulationpod.json
+$flat | Set-Content -Path "$archive_name.json"
-
 
 # load json into memory, convert to psobject
-$json = Get-Content -Path YT_theregulationpod.json | ConvertFrom-Json
+$json = Get-Content -Path "$archive_name.json" | ConvertFrom-Json
 
 # Load all .mp4 files into memory once at the start
 $mp4Files = Get-ChildItem -Path $folderPath -Recurse -Filter "*.mp4"
 
-
+ForEach ($obj in ($json | Sort-Object playlist_index -Descending | Select-Object -First 10)) {
-ForEach ($obj in $json) {
     # Initialize $fileFound as null for each iteration
     $fileFound = $false
 
@@ -148,10 +189,10 @@ ForEach ($obj in $json) {
     $videoUrl = $obj.webpage_url
     $Title = $obj.title
     $Id = $obj.id
-
+    $extractor = $obj.extractor
 
     # Log the start of the process
-    Write-Verbose -Message "Start $Id ##### $Title" -Verbose
+    Write-Verbose -Message "Start $extractor-$Id ##### $Title" -Verbose
 
     # Check if the title matches any reject term using wildcards
     $reject_matches = $rejects | Where-Object { $Title -like "*$($_)*" }
@@ -161,63 +202,21 @@ ForEach ($obj in $json) {
     }
 
     # downloaded already?
-    # Regular expression to match the ID after "youtube-" and before the closing bracket
+    # Regular expression to match the ID, supporting both YouTube and Patreon prefixes
-    $regex = "(?i)youtube-$Id"
+    # (?:youtube-|patreon-) creates a non-capturing group that matches either prefix
+    $regex = "(?i)(?:youtube-|patreon-)$([regex]::Escape($Id))" 
 
     # Check if any file already exists that matches the video ID
     $fileFound = $mp4Files | Where-Object { $_.Name -match $regex }
 
-
     # If no file is found, notify the user
     if ($fileFound) {
         Write-Warning "File found with the YouTube ID: $Id"
     }
-    Else {
+    Else {  
-        # Check if this is a YouTube Short by getting video info (aspect ratio or duration)
-        Write-Verbose -Message "Checking if video is a Short: $Title" -Verbose
-        $isShort = $false
-        $videoInfo = D:\yt-dlp\yt-dlp.exe -j $videoUrl
-        $videoInfoObj = $videoInfo | ConvertFrom-Json
-
-
-        if ($videoUrl -like "*shorts*") {
-            Write-Verbose "Identified as Shorts video because the URL contains 'shorts': $Id ##### $Title" -Verbose
-            $isShort = $true
-        }
-        Elseif ($videoInfoObj.duration -and $videoInfoObj.duration -lt 60) {
-            # Determine if this is a short based on duration
-            Write-Verbose "Identified as Shorts video because the duration is 60 seconds or less: $Id ##### $Title" -Verbose
-            $isShort = $true
-        } 
-        elseif ($videoInfoObj.width -and $videoInfoObj.height) {
-            # Calculate aspect ratio (height/width for vertical videos)
-            $aspectRatio = [double]$videoInfoObj.height / [double]$videoInfoObj.width
-            if ($aspectRatio -gt 1.5) {
-                # Vertical video ( > 1.5:1)
-                $isShort = $true
-            }
-        }
-            
-        # Set output path based on whether it's a short or not
-        if ($isShort) {
-            Write-Verbose -Message "Video detected as Short: $Title" -Verbose
-            # Note: This path must be determined before running the tracking check/download
-            $outputPath = Join-Path $folderPath "YouTube Shorts/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
-            $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
-            
-            # Check if ID is in the tracking JSON file
-            $idAlreadyTracked = Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$false 
-
-            if ($idAlreadyTracked) {
-                Write-Warning "Skipping Short $Id ##### $Title because it was previously tracked and deleted."
-                continue # Skip the download attempt entirely
-            }
-        }
-        else {
-            Write-Verbose -Message "Video detected as regular: $Title" -Verbose
-            $outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y,upload_date>%y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
-        }
 
+        $outputPath = Join-Path $folderPath "Unsorted/Season %(upload_date>%y|Unknown)s/%(upload_date>%m|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
+        
         # Execute yt-dlp command for non-rejected videos
         try {
             Write-Warning -Message "Dwnld $Id ##### $Title"
@@ -248,7 +247,7 @@ ForEach ($obj in $json) {
         }
 
         if ($isShort -and $DownloadSuccessful) {
-             $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
+            $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
             Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$true 
         }
     }

yt-dlp/Download-RegulationPodcast_NEW.ps1

@@ -3,7 +3,6 @@
 $URL = "https://www.youtube.com/@theregulationpod"
 $folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$'
 
-
 $rejects = @()
 $RejectString = $rejects -join '|'
 
@@ -29,6 +28,7 @@ function Download-YTDLP {
         $videoUrl,
         "--write-info-json",
         "--write-thumbnail",
+        "--embed-metadata".
         "--convert-thumbnails", "jpg",
         "--no-progress", 
         "-t", "sleep",
@@ -66,10 +66,10 @@ function Download-YTDLP {
 # Function to manage Short ID tracking (NEW)
 function Manage-ShortTracking {
     Param(
-        [Parameter(Mandatory=$true)]
+        [Parameter(Mandatory = $true)]
         [string]$ShortDirPath,
 
-        [Parameter(Mandatory=$true)]
+        [Parameter(Mandatory = $true)]
         [string]$VideoId,
 
         [switch]$IsNewDownload
@@ -85,8 +85,9 @@ function Manage-ShortTracking {
                 if ($existingIds -contains $VideoId) {
                     return $true # Video ID found, should skip download
                 }
-            } catch {
+            }
-                 Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
+            catch {
+                Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
             }
         }
 
@@ -99,47 +100,86 @@ function Manage-ShortTracking {
                     $newIds = $existingIds + $VideoId
                     $newIds | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile
                     Write-Verbose "Successfully added ID '$VideoId' to shorts tracking."
-                } else {
+                }
-                     Write-Verbose "ID '$VideoId' already exists in tracking file."
+                else {
+                    Write-Verbose "ID '$VideoId' already exists in tracking file."
                 }
 
-            } else {
+            }
-                 # File does not exist, create it
+            else {
+                # File does not exist, create it
                 @($VideoId) | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile
                 Write-Verbose "Created and added ID '$VideoId' to shorts tracking file."
             }
-        } catch {
+        }
+        catch {
             Write-Error "Failed to write to shorts tracking file: $($_.Exception.Message)"
         }
-    } else { 
+    }
+    else { 
         # Check mode (only needed when checking before download)
-         if (Test-Path $trackingFile) {
+        if (Test-Path $trackingFile) {
             try {
                 $existingIds = Get-Content -Path $trackingFile | ConvertFrom-Json
                 return $existingIds -contains $VideoId # Returns True if ID is found
-            } catch {
+            }
-                 Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
+            catch {
-                 return $false # Assume safe to download if parsing fails
+                Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
+                return $false # Assume safe to download if parsing fails
             }
         }
         return $false # No tracking file exists, assume safe to download
     }
 }
 
+function Parse-URLToIdentifier {
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Url
+    )
+
+    $platform = ""
+    $name = ""
+
+    # 1. Determine the Platform
+    if ($Url -match "patreon\.com") {
+        $platform = "patreon"
+        # For Patreon, take the last segment of the URL path (e.g., TheRegulationPod)
+        $name = ($Url -split '/')[-1]
+    }
+    elseif ($Url -match "youtube.com") {
+        $platform = "youtube"
+        # For YouTube, extract everything after the "@" symbol
+        if ($Url -match '@(.*)') {
+            $name = $Matches[1]
+        }
+        else {
+            # Handle cases where @ might not be present (though unlikely for channel URLs)
+            $name = "UnknownChannel" 
+        }
+    }
+    else {
+        Write-Error "Unsupported URL platform."
+        return $null
+    }
+
+    # Construct and return the final identifier
+    return "${platform}_${name}"
+}
+
+$archive_name = Parse-URLToIdentifier -Url $Url
 
 # Download flat playlist
 Write-Verbose -Message "Download flat playlist"
 $flat = D:\yt-dlp\yt-dlp.exe -j --flat-playlist $URL
-$flat | Set-Content -Path YouTube_theregulationpod.json
+$flat | Set-Content -Path "$archive_name.json"
-
 
 # load json into memory, convert to psobject
-$json = Get-Content -Path YouTube_theregulationpod.json | ConvertFrom-Json
+$json = Get-Content -Path "$archive_name.json" | ConvertFrom-Json
 
 # Load all .mp4 files into memory once at the start
 $mp4Files = Get-ChildItem -Path $folderPath -Recurse -Filter "*.mp4"
 
-
 ForEach ($obj in $json) {
     # Initialize $fileFound as null for each iteration
     $fileFound = $false
@@ -149,7 +189,6 @@ ForEach ($obj in $json) {
     $Title = $obj.title
     $Id = $obj.id
 
-
     # Log the start of the process
     Write-Verbose -Message "Start $Id ##### $Title" -Verbose
 
@@ -161,25 +200,26 @@ ForEach ($obj in $json) {
     }
 
     # downloaded already?
-    # Regular expression to match the ID after "youtube-" and before the closing bracket
+    # Regular expression to match the ID, supporting both YouTube and Patreon prefixes
-    $regex = "(?i)youtube-$Id"
+    # (?:youtube-|patreon-) creates a non-capturing group that matches either prefix
+    $regex = "(?i)(?:youtube-|patreon-)$([regex]::Escape($Id))" 
 
     # Check if any file already exists that matches the video ID
     $fileFound = $mp4Files | Where-Object { $_.Name -match $regex }
 
 
+
     # If no file is found, notify the user
     if ($fileFound) {
         Write-Warning "File found with the YouTube ID: $Id"
     }
     Else {
-        # Check if this is a YouTube Short by getting video info (aspect ratio or duration)
+        <# Check if this is a YouTube Short by getting video info (aspect ratio or duration)
         Write-Verbose -Message "Checking if video is a Short: $Title" -Verbose
         $isShort = $false
         $videoInfo = D:\yt-dlp\yt-dlp.exe -j $videoUrl
         $videoInfoObj = $videoInfo | ConvertFrom-Json
 
-
         if ($videoUrl -like "*shorts*") {
             Write-Verbose "Identified as Shorts video because the URL contains 'shorts': $Id ##### $Title" -Verbose
             $isShort = $true
@@ -188,7 +228,7 @@ ForEach ($obj in $json) {
             # Determine if this is a short based on duration
             Write-Verbose "Identified as Shorts video because the duration is 60 seconds or less: $Id ##### $Title" -Verbose
             $isShort = $true
-        } 
+        } <#
         elseif ($videoInfoObj.width -and $videoInfoObj.height) {
             # Calculate aspect ratio (height/width for vertical videos)
             $aspectRatio = [double]$videoInfoObj.height / [double]$videoInfoObj.width
@@ -196,13 +236,13 @@ ForEach ($obj in $json) {
                 # Vertical video ( > 1.5:1)
                 $isShort = $true
             }
-        }
+        }#>
             
         # Set output path based on whether it's a short or not
         if ($isShort) {
             Write-Verbose -Message "Video detected as Short: $Title" -Verbose
             # Note: This path must be determined before running the tracking check/download
-            $outputPath = Join-Path $folderPath "YouTube Shorts/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
+            $outputPath = Join-Path $folderPath "Unsorted/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
             $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
             
             # Check if ID is in the tracking JSON file
@@ -215,7 +255,7 @@ ForEach ($obj in $json) {
         }
         else {
             Write-Verbose -Message "Video detected as regular: $Title" -Verbose
-            $outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y,upload_date>%y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
+            $outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y|Unknown)s/%(release_date>%m|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
         }
 
         # Execute yt-dlp command for non-rejected videos
@@ -248,7 +288,7 @@ ForEach ($obj in $json) {
         }
 
         if ($isShort -and $DownloadSuccessful) {
-             $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
+            $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
             Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$true 
         }
     }