diff --git a/_Notes/Proxmox_SwapDisk.md b/_Notes/Proxmox_SwapDisk.md new file mode 100644 index 0000000..e69de29 diff --git a/yt-dlp/Download-RegulationGameplay_NEW.ps1 b/yt-dlp/Download-RegulationGameplay_NEW.ps1 index fae34dc..1086c2f 100644 --- a/yt-dlp/Download-RegulationGameplay_NEW.ps1 +++ b/yt-dlp/Download-RegulationGameplay_NEW.ps1 @@ -3,7 +3,6 @@ $URL = "https://www.youtube.com/@TheRegulationPodcast" $folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$' - $rejects = @() $RejectString = $rejects -join '|' diff --git a/yt-dlp/Download-RegulationPatreon_NEW.ps1 b/yt-dlp/Download-RegulationPatreon_NEW.ps1 index e15f96a..34d32df 100644 --- a/yt-dlp/Download-RegulationPatreon_NEW.ps1 +++ b/yt-dlp/Download-RegulationPatreon_NEW.ps1 @@ -1,9 +1,8 @@ -## YT Regulation Podcast +# YT Regulation Patreon # Set URL and Location -$URL = "https://www.youtube.com/@theregulationpod" +$URL = "https://www.patreon.com/cw/TheRegulationPod" $folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$' - $rejects = @() $RejectString = $rejects -join '|' @@ -26,18 +25,20 @@ function Download-YTDLP { # Start building an array of arguments using strict comma separation (Recommended format) $arguments = @( - $videoUrl, + "$videoUrl", "--write-info-json", "--write-thumbnail", + "--embed-metadata", "--convert-thumbnails", "jpg", "--no-progress", "-t", "sleep", "--rate-limit", "5M", - "-o", $OutputPath, + "-o", "$OutputPath", '-f', "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best", "-S", "vcodec:h264", - "--reject-title", $RejectString, - "--playlist-reverse" + "--reject-title", "$RejectString", + "--playlist-reverse", + "--download-archive", "ytdlp_archive.txt" ) # --- Conditional Logic Applied Here --- @@ -66,10 +67,10 @@ function Download-YTDLP { # Function to manage Short ID tracking (NEW) function Manage-ShortTracking { Param( - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string]$ShortDirPath, - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string]$VideoId, [switch]$IsNewDownload @@ -85,8 +86,9 @@ function Manage-ShortTracking { if ($existingIds -contains $VideoId) { return $true # Video ID found, should skip download } - } catch { - Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)" + } + catch { + Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)" } } @@ -99,48 +101,87 @@ function Manage-ShortTracking { $newIds = $existingIds + $VideoId $newIds | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile Write-Verbose "Successfully added ID '$VideoId' to shorts tracking." - } else { - Write-Verbose "ID '$VideoId' already exists in tracking file." + } + else { + Write-Verbose "ID '$VideoId' already exists in tracking file." } - } else { - # File does not exist, create it + } + else { + # File does not exist, create it @($VideoId) | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile Write-Verbose "Created and added ID '$VideoId' to shorts tracking file." } - } catch { + } + catch { Write-Error "Failed to write to shorts tracking file: $($_.Exception.Message)" } - } else { + } + else { # Check mode (only needed when checking before download) - if (Test-Path $trackingFile) { + if (Test-Path $trackingFile) { try { $existingIds = Get-Content -Path $trackingFile | ConvertFrom-Json return $existingIds -contains $VideoId # Returns True if ID is found - } catch { - Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)" - return $false # Assume safe to download if parsing fails + } + catch { + Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)" + return $false # Assume safe to download if parsing fails } } return $false # No tracking file exists, assume safe to download } } +function Parse-URLToIdentifier { + param( + [Parameter(Mandatory = $true)] + [string]$Url + ) + + $platform = "" + $name = "" + + # 1. Determine the Platform + if ($Url -match "patreon\.com") { + $platform = "patreon" + # For Patreon, take the last segment of the URL path (e.g., TheRegulationPod) + $name = ($Url -split '/')[-1] + } + elseif ($Url -match "youtube.com") { + $platform = "youtube" + # For YouTube, extract everything after the "@" symbol + if ($Url -match '@(.*)') { + $name = $Matches[1] + } + else { + # Handle cases where @ might not be present (though unlikely for channel URLs) + $name = "UnknownChannel" + } + } + else { + Write-Error "Unsupported URL platform." + return $null + } + + # Construct and return the final identifier + return "${platform}_${name}" +} + +$archive_name = Parse-URLToIdentifier -Url $Url # Download flat playlist Write-Verbose -Message "Download flat playlist" $flat = D:\yt-dlp\yt-dlp.exe -j --flat-playlist $URL -$flat | Set-Content -Path YT_theregulationpod.json - +$flat | Set-Content -Path "$archive_name.json" # load json into memory, convert to psobject -$json = Get-Content -Path YT_theregulationpod.json | ConvertFrom-Json +$json = Get-Content -Path "$archive_name.json" | ConvertFrom-Json # Load all .mp4 files into memory once at the start $mp4Files = Get-ChildItem -Path $folderPath -Recurse -Filter "*.mp4" - -ForEach ($obj in $json) { +ForEach ($obj in ($json | Sort-Object playlist_index -Descending | Select-Object -First 10)) { # Initialize $fileFound as null for each iteration $fileFound = $false @@ -148,10 +189,10 @@ ForEach ($obj in $json) { $videoUrl = $obj.webpage_url $Title = $obj.title $Id = $obj.id - + $extractor = $obj.extractor # Log the start of the process - Write-Verbose -Message "Start $Id ##### $Title" -Verbose + Write-Verbose -Message "Start $extractor-$Id ##### $Title" -Verbose # Check if the title matches any reject term using wildcards $reject_matches = $rejects | Where-Object { $Title -like "*$($_)*" } @@ -161,63 +202,21 @@ ForEach ($obj in $json) { } # downloaded already? - # Regular expression to match the ID after "youtube-" and before the closing bracket - $regex = "(?i)youtube-$Id" + # Regular expression to match the ID, supporting both YouTube and Patreon prefixes + # (?:youtube-|patreon-) creates a non-capturing group that matches either prefix + $regex = "(?i)(?:youtube-|patreon-)$([regex]::Escape($Id))" # Check if any file already exists that matches the video ID $fileFound = $mp4Files | Where-Object { $_.Name -match $regex } - # If no file is found, notify the user if ($fileFound) { Write-Warning "File found with the YouTube ID: $Id" } - Else { - # Check if this is a YouTube Short by getting video info (aspect ratio or duration) - Write-Verbose -Message "Checking if video is a Short: $Title" -Verbose - $isShort = $false - $videoInfo = D:\yt-dlp\yt-dlp.exe -j $videoUrl - $videoInfoObj = $videoInfo | ConvertFrom-Json - - - if ($videoUrl -like "*shorts*") { - Write-Verbose "Identified as Shorts video because the URL contains 'shorts': $Id ##### $Title" -Verbose - $isShort = $true - } - Elseif ($videoInfoObj.duration -and $videoInfoObj.duration -lt 60) { - # Determine if this is a short based on duration - Write-Verbose "Identified as Shorts video because the duration is 60 seconds or less: $Id ##### $Title" -Verbose - $isShort = $true - } - elseif ($videoInfoObj.width -and $videoInfoObj.height) { - # Calculate aspect ratio (height/width for vertical videos) - $aspectRatio = [double]$videoInfoObj.height / [double]$videoInfoObj.width - if ($aspectRatio -gt 1.5) { - # Vertical video ( > 1.5:1) - $isShort = $true - } - } - - # Set output path based on whether it's a short or not - if ($isShort) { - Write-Verbose -Message "Video detected as Short: $Title" -Verbose - # Note: This path must be determined before running the tracking check/download - $outputPath = Join-Path $folderPath "YouTube Shorts/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s" - $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...) - - # Check if ID is in the tracking JSON file - $idAlreadyTracked = Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$false - - if ($idAlreadyTracked) { - Write-Warning "Skipping Short $Id ##### $Title because it was previously tracked and deleted." - continue # Skip the download attempt entirely - } - } - else { - Write-Verbose -Message "Video detected as regular: $Title" -Verbose - $outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y,upload_date>%y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s" - } + Else { + $outputPath = Join-Path $folderPath "Unsorted/Season %(upload_date>%y|Unknown)s/%(upload_date>%m|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s" + # Execute yt-dlp command for non-rejected videos try { Write-Warning -Message "Dwnld $Id ##### $Title" @@ -248,7 +247,7 @@ ForEach ($obj in $json) { } if ($isShort -and $DownloadSuccessful) { - $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...) + $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...) Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$true } } diff --git a/yt-dlp/Download-RegulationPodcast_NEW.ps1 b/yt-dlp/Download-RegulationPodcast_NEW.ps1 index f42edfb..f9f4cbb 100644 --- a/yt-dlp/Download-RegulationPodcast_NEW.ps1 +++ b/yt-dlp/Download-RegulationPodcast_NEW.ps1 @@ -3,7 +3,6 @@ $URL = "https://www.youtube.com/@theregulationpod" $folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$' - $rejects = @() $RejectString = $rejects -join '|' @@ -29,6 +28,7 @@ function Download-YTDLP { $videoUrl, "--write-info-json", "--write-thumbnail", + "--embed-metadata". "--convert-thumbnails", "jpg", "--no-progress", "-t", "sleep", @@ -66,10 +66,10 @@ function Download-YTDLP { # Function to manage Short ID tracking (NEW) function Manage-ShortTracking { Param( - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string]$ShortDirPath, - [Parameter(Mandatory=$true)] + [Parameter(Mandatory = $true)] [string]$VideoId, [switch]$IsNewDownload @@ -85,8 +85,9 @@ function Manage-ShortTracking { if ($existingIds -contains $VideoId) { return $true # Video ID found, should skip download } - } catch { - Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)" + } + catch { + Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)" } } @@ -99,47 +100,86 @@ function Manage-ShortTracking { $newIds = $existingIds + $VideoId $newIds | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile Write-Verbose "Successfully added ID '$VideoId' to shorts tracking." - } else { - Write-Verbose "ID '$VideoId' already exists in tracking file." + } + else { + Write-Verbose "ID '$VideoId' already exists in tracking file." } - } else { - # File does not exist, create it + } + else { + # File does not exist, create it @($VideoId) | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile Write-Verbose "Created and added ID '$VideoId' to shorts tracking file." } - } catch { + } + catch { Write-Error "Failed to write to shorts tracking file: $($_.Exception.Message)" } - } else { + } + else { # Check mode (only needed when checking before download) - if (Test-Path $trackingFile) { + if (Test-Path $trackingFile) { try { $existingIds = Get-Content -Path $trackingFile | ConvertFrom-Json return $existingIds -contains $VideoId # Returns True if ID is found - } catch { - Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)" - return $false # Assume safe to download if parsing fails + } + catch { + Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)" + return $false # Assume safe to download if parsing fails } } return $false # No tracking file exists, assume safe to download } } +function Parse-URLToIdentifier { + param( + [Parameter(Mandatory = $true)] + [string]$Url + ) + + $platform = "" + $name = "" + + # 1. Determine the Platform + if ($Url -match "patreon\.com") { + $platform = "patreon" + # For Patreon, take the last segment of the URL path (e.g., TheRegulationPod) + $name = ($Url -split '/')[-1] + } + elseif ($Url -match "youtube.com") { + $platform = "youtube" + # For YouTube, extract everything after the "@" symbol + if ($Url -match '@(.*)') { + $name = $Matches[1] + } + else { + # Handle cases where @ might not be present (though unlikely for channel URLs) + $name = "UnknownChannel" + } + } + else { + Write-Error "Unsupported URL platform." + return $null + } + + # Construct and return the final identifier + return "${platform}_${name}" +} + +$archive_name = Parse-URLToIdentifier -Url $Url # Download flat playlist Write-Verbose -Message "Download flat playlist" $flat = D:\yt-dlp\yt-dlp.exe -j --flat-playlist $URL -$flat | Set-Content -Path YouTube_theregulationpod.json - +$flat | Set-Content -Path "$archive_name.json" # load json into memory, convert to psobject -$json = Get-Content -Path YouTube_theregulationpod.json | ConvertFrom-Json +$json = Get-Content -Path "$archive_name.json" | ConvertFrom-Json # Load all .mp4 files into memory once at the start $mp4Files = Get-ChildItem -Path $folderPath -Recurse -Filter "*.mp4" - ForEach ($obj in $json) { # Initialize $fileFound as null for each iteration $fileFound = $false @@ -149,7 +189,6 @@ ForEach ($obj in $json) { $Title = $obj.title $Id = $obj.id - # Log the start of the process Write-Verbose -Message "Start $Id ##### $Title" -Verbose @@ -161,25 +200,26 @@ ForEach ($obj in $json) { } # downloaded already? - # Regular expression to match the ID after "youtube-" and before the closing bracket - $regex = "(?i)youtube-$Id" + # Regular expression to match the ID, supporting both YouTube and Patreon prefixes + # (?:youtube-|patreon-) creates a non-capturing group that matches either prefix + $regex = "(?i)(?:youtube-|patreon-)$([regex]::Escape($Id))" # Check if any file already exists that matches the video ID $fileFound = $mp4Files | Where-Object { $_.Name -match $regex } + # If no file is found, notify the user if ($fileFound) { Write-Warning "File found with the YouTube ID: $Id" } Else { - # Check if this is a YouTube Short by getting video info (aspect ratio or duration) + <# Check if this is a YouTube Short by getting video info (aspect ratio or duration) Write-Verbose -Message "Checking if video is a Short: $Title" -Verbose $isShort = $false $videoInfo = D:\yt-dlp\yt-dlp.exe -j $videoUrl $videoInfoObj = $videoInfo | ConvertFrom-Json - if ($videoUrl -like "*shorts*") { Write-Verbose "Identified as Shorts video because the URL contains 'shorts': $Id ##### $Title" -Verbose $isShort = $true @@ -188,7 +228,7 @@ ForEach ($obj in $json) { # Determine if this is a short based on duration Write-Verbose "Identified as Shorts video because the duration is 60 seconds or less: $Id ##### $Title" -Verbose $isShort = $true - } + } <# elseif ($videoInfoObj.width -and $videoInfoObj.height) { # Calculate aspect ratio (height/width for vertical videos) $aspectRatio = [double]$videoInfoObj.height / [double]$videoInfoObj.width @@ -196,13 +236,13 @@ ForEach ($obj in $json) { # Vertical video ( > 1.5:1) $isShort = $true } - } + }#> # Set output path based on whether it's a short or not if ($isShort) { Write-Verbose -Message "Video detected as Short: $Title" -Verbose # Note: This path must be determined before running the tracking check/download - $outputPath = Join-Path $folderPath "YouTube Shorts/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s" + $outputPath = Join-Path $folderPath "Unsorted/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s" $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...) # Check if ID is in the tracking JSON file @@ -215,7 +255,7 @@ ForEach ($obj in $json) { } else { Write-Verbose -Message "Video detected as regular: $Title" -Verbose - $outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y,upload_date>%y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s" + $outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y|Unknown)s/%(release_date>%m|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s" } # Execute yt-dlp command for non-rejected videos @@ -248,7 +288,7 @@ ForEach ($obj in $json) { } if ($isShort -and $DownloadSuccessful) { - $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...) + $shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...) Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$true } }