zmeier/Homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DL updates

Browse Source
This commit is contained in:
Zack Meier
2026-04-28 09:51:21 -05:00
parent 18d5913d5e
commit 57c717aff4
4 changed files with 145 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yt-dlp/Download-RegulationGameplay_NEW.ps1
-1
View File
@@ -3,7 +3,6 @@
$URL = "https://www.youtube.com/@TheRegulationPodcast"
$folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$'
$rejects = @()
$RejectString = $rejects -join '|'
yt-dlp/Download-RegulationPatreon_NEW.ps1
+76 -77
View File
@@ -1,9 +1,8 @@
## YT Regulation Podcast
# YT Regulation Patreon
# Set URL and Location
$URL = "https://www.youtube.com/@theregulationpod"
$URL = "https://www.patreon.com/cw/TheRegulationPod"
$folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$'
$rejects = @()
$RejectString = $rejects -join '|'
@@ -26,18 +25,20 @@ function Download-YTDLP {
# Start building an array of arguments using strict comma separation (Recommended format)
$arguments = @(
$videoUrl,
"$videoUrl",
"--write-info-json",
"--write-thumbnail",
"--embed-metadata",
"--convert-thumbnails", "jpg",
"--no-progress",
"-t", "sleep",
"--rate-limit", "5M",
"-o", $OutputPath,
"-o", "$OutputPath",
'-f', "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best",
"-S", "vcodec:h264",
"--reject-title", $RejectString,
"--playlist-reverse"
"--reject-title", "$RejectString",
"--playlist-reverse",
"--download-archive", "ytdlp_archive.txt"
)
# --- Conditional Logic Applied Here ---
@@ -66,10 +67,10 @@ function Download-YTDLP {
# Function to manage Short ID tracking (NEW)
function Manage-ShortTracking {
Param(
[Parameter(Mandatory=$true)]
[Parameter(Mandatory = $true)]
[string]$ShortDirPath,
[Parameter(Mandatory=$true)]
[Parameter(Mandatory = $true)]
[string]$VideoId,
[switch]$IsNewDownload
@@ -85,8 +86,9 @@ function Manage-ShortTracking {
if ($existingIds -contains $VideoId) {
return $true # Video ID found, should skip download
}
} catch {
Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
}
catch {
Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
}
}
@@ -99,48 +101,87 @@ function Manage-ShortTracking {
$newIds = $existingIds + $VideoId
$newIds | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile
Write-Verbose "Successfully added ID '$VideoId' to shorts tracking."
} else {
Write-Verbose "ID '$VideoId' already exists in tracking file."
}
else {
Write-Verbose "ID '$VideoId' already exists in tracking file."
}
} else {
# File does not exist, create it
}
else {
# File does not exist, create it
@($VideoId) | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile
Write-Verbose "Created and added ID '$VideoId' to shorts tracking file."
}
} catch {
}
catch {
Write-Error "Failed to write to shorts tracking file: $($_.Exception.Message)"
}
} else {
}
else {
# Check mode (only needed when checking before download)
if (Test-Path $trackingFile) {
if (Test-Path $trackingFile) {
try {
$existingIds = Get-Content -Path $trackingFile | ConvertFrom-Json
return $existingIds -contains $VideoId # Returns True if ID is found
} catch {
Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
return $false # Assume safe to download if parsing fails
}
catch {
Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
return $false # Assume safe to download if parsing fails
}
}
return $false # No tracking file exists, assume safe to download
}
}
function Parse-URLToIdentifier {
param(
[Parameter(Mandatory = $true)]
[string]$Url
)
$platform = ""
$name = ""
# 1. Determine the Platform
if ($Url -match "patreon\.com") {
$platform = "patreon"
# For Patreon, take the last segment of the URL path (e.g., TheRegulationPod)
$name = ($Url -split '/')[-1]
}
elseif ($Url -match "youtube.com") {
$platform = "youtube"
# For YouTube, extract everything after the "@" symbol
if ($Url -match '@(.*)') {
$name = $Matches[1]
}
else {
# Handle cases where @ might not be present (though unlikely for channel URLs)
$name = "UnknownChannel"
}
}
else {
Write-Error "Unsupported URL platform."
return $null
}
# Construct and return the final identifier
return "${platform}_${name}"
}
$archive_name = Parse-URLToIdentifier -Url $Url
# Download flat playlist
Write-Verbose -Message "Download flat playlist"
$flat = D:\yt-dlp\yt-dlp.exe -j --flat-playlist $URL
$flat | Set-Content -Path YT_theregulationpod.json
$flat | Set-Content -Path "$archive_name.json"
# load json into memory, convert to psobject
$json = Get-Content -Path YT_theregulationpod.json | ConvertFrom-Json
$json = Get-Content -Path "$archive_name.json" | ConvertFrom-Json
# Load all .mp4 files into memory once at the start
$mp4Files = Get-ChildItem -Path $folderPath -Recurse -Filter "*.mp4"
ForEach ($obj in $json) {
ForEach ($obj in ($json | Sort-Object playlist_index -Descending | Select-Object -First 10)) {
# Initialize $fileFound as null for each iteration
$fileFound = $false
@@ -148,10 +189,10 @@ ForEach ($obj in $json) {
$videoUrl = $obj.webpage_url
$Title = $obj.title
$Id = $obj.id
$extractor = $obj.extractor
# Log the start of the process
Write-Verbose -Message "Start $Id ##### $Title" -Verbose
Write-Verbose -Message "Start $extractor-$Id ##### $Title" -Verbose
# Check if the title matches any reject term using wildcards
$reject_matches = $rejects | Where-Object { $Title -like "*$($_)*" }
@@ -161,63 +202,21 @@ ForEach ($obj in $json) {
}
# downloaded already?
# Regular expression to match the ID after "youtube-" and before the closing bracket
$regex = "(?i)youtube-$Id"
# Regular expression to match the ID, supporting both YouTube and Patreon prefixes
# (?:youtube-|patreon-) creates a non-capturing group that matches either prefix
$regex = "(?i)(?:youtube-|patreon-)$([regex]::Escape($Id))"
# Check if any file already exists that matches the video ID
$fileFound = $mp4Files | Where-Object { $_.Name -match $regex }
# If no file is found, notify the user
if ($fileFound) {
Write-Warning "File found with the YouTube ID: $Id"
}
Else {
# Check if this is a YouTube Short by getting video info (aspect ratio or duration)
Write-Verbose -Message "Checking if video is a Short: $Title" -Verbose
$isShort = $false
$videoInfo = D:\yt-dlp\yt-dlp.exe -j $videoUrl
$videoInfoObj = $videoInfo | ConvertFrom-Json
if ($videoUrl -like "*shorts*") {
Write-Verbose "Identified as Shorts video because the URL contains 'shorts': $Id ##### $Title" -Verbose
$isShort = $true
}
Elseif ($videoInfoObj.duration -and $videoInfoObj.duration -lt 60) {
# Determine if this is a short based on duration
Write-Verbose "Identified as Shorts video because the duration is 60 seconds or less: $Id ##### $Title" -Verbose
$isShort = $true
}
elseif ($videoInfoObj.width -and $videoInfoObj.height) {
# Calculate aspect ratio (height/width for vertical videos)
$aspectRatio = [double]$videoInfoObj.height / [double]$videoInfoObj.width
if ($aspectRatio -gt 1.5) {
# Vertical video ( > 1.5:1)
$isShort = $true
}
}
# Set output path based on whether it's a short or not
if ($isShort) {
Write-Verbose -Message "Video detected as Short: $Title" -Verbose
# Note: This path must be determined before running the tracking check/download
$outputPath = Join-Path $folderPath "YouTube Shorts/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
$shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
# Check if ID is in the tracking JSON file
$idAlreadyTracked = Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$false
if ($idAlreadyTracked) {
Write-Warning "Skipping Short $Id ##### $Title because it was previously tracked and deleted."
continue # Skip the download attempt entirely
}
}
else {
Write-Verbose -Message "Video detected as regular: $Title" -Verbose
$outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y,upload_date>%y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
}
Else {
$outputPath = Join-Path $folderPath "Unsorted/Season %(upload_date>%y|Unknown)s/%(upload_date>%m|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
# Execute yt-dlp command for non-rejected videos
try {
Write-Warning -Message "Dwnld $Id ##### $Title"
@@ -248,7 +247,7 @@ ForEach ($obj in $json) {
}
if ($isShort -and $DownloadSuccessful) {
$shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
$shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$true
}
}
yt-dlp/Download-RegulationPodcast_NEW.ps1
+69 -29
View File
@@ -3,7 +3,6 @@
$URL = "https://www.youtube.com/@theregulationpod"
$folderPath = '\\truenas3.kwyjibo.info\RegulationPodcast$'
$rejects = @()
$RejectString = $rejects -join '|'
@@ -29,6 +28,7 @@ function Download-YTDLP {
$videoUrl,
"--write-info-json",
"--write-thumbnail",
"--embed-metadata".
"--convert-thumbnails", "jpg",
"--no-progress",
"-t", "sleep",
@@ -66,10 +66,10 @@ function Download-YTDLP {
# Function to manage Short ID tracking (NEW)
function Manage-ShortTracking {
Param(
[Parameter(Mandatory=$true)]
[Parameter(Mandatory = $true)]
[string]$ShortDirPath,
[Parameter(Mandatory=$true)]
[Parameter(Mandatory = $true)]
[string]$VideoId,
[switch]$IsNewDownload
@@ -85,8 +85,9 @@ function Manage-ShortTracking {
if ($existingIds -contains $VideoId) {
return $true # Video ID found, should skip download
}
} catch {
Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
}
catch {
Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
}
}
@@ -99,47 +100,86 @@ function Manage-ShortTracking {
$newIds = $existingIds + $VideoId
$newIds | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile
Write-Verbose "Successfully added ID '$VideoId' to shorts tracking."
} else {
Write-Verbose "ID '$VideoId' already exists in tracking file."
}
else {
Write-Verbose "ID '$VideoId' already exists in tracking file."
}
} else {
# File does not exist, create it
}
else {
# File does not exist, create it
@($VideoId) | ConvertTo-Json -Depth 3 | Set-Content -Path $trackingFile
Write-Verbose "Created and added ID '$VideoId' to shorts tracking file."
}
} catch {
}
catch {
Write-Error "Failed to write to shorts tracking file: $($_.Exception.Message)"
}
} else {
}
else {
# Check mode (only needed when checking before download)
if (Test-Path $trackingFile) {
if (Test-Path $trackingFile) {
try {
$existingIds = Get-Content -Path $trackingFile | ConvertFrom-Json
return $existingIds -contains $VideoId # Returns True if ID is found
} catch {
Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
return $false # Assume safe to download if parsing fails
}
catch {
Write-Error "Could not read or parse existing shorts tracking file: $($_.Exception.Message)"
return $false # Assume safe to download if parsing fails
}
}
return $false # No tracking file exists, assume safe to download
}
}
function Parse-URLToIdentifier {
param(
[Parameter(Mandatory = $true)]
[string]$Url
)
$platform = ""
$name = ""
# 1. Determine the Platform
if ($Url -match "patreon\.com") {
$platform = "patreon"
# For Patreon, take the last segment of the URL path (e.g., TheRegulationPod)
$name = ($Url -split '/')[-1]
}
elseif ($Url -match "youtube.com") {
$platform = "youtube"
# For YouTube, extract everything after the "@" symbol
if ($Url -match '@(.*)') {
$name = $Matches[1]
}
else {
# Handle cases where @ might not be present (though unlikely for channel URLs)
$name = "UnknownChannel"
}
}
else {
Write-Error "Unsupported URL platform."
return $null
}
# Construct and return the final identifier
return "${platform}_${name}"
}
$archive_name = Parse-URLToIdentifier -Url $Url
# Download flat playlist
Write-Verbose -Message "Download flat playlist"
$flat = D:\yt-dlp\yt-dlp.exe -j --flat-playlist $URL
$flat | Set-Content -Path YouTube_theregulationpod.json
$flat | Set-Content -Path "$archive_name.json"
# load json into memory, convert to psobject
$json = Get-Content -Path YouTube_theregulationpod.json | ConvertFrom-Json
$json = Get-Content -Path "$archive_name.json" | ConvertFrom-Json
# Load all .mp4 files into memory once at the start
$mp4Files = Get-ChildItem -Path $folderPath -Recurse -Filter "*.mp4"
ForEach ($obj in $json) {
# Initialize $fileFound as null for each iteration
$fileFound = $false
@@ -149,7 +189,6 @@ ForEach ($obj in $json) {
$Title = $obj.title
$Id = $obj.id
# Log the start of the process
Write-Verbose -Message "Start $Id ##### $Title" -Verbose
@@ -161,25 +200,26 @@ ForEach ($obj in $json) {
}
# downloaded already?
# Regular expression to match the ID after "youtube-" and before the closing bracket
$regex = "(?i)youtube-$Id"
# Regular expression to match the ID, supporting both YouTube and Patreon prefixes
# (?:youtube-|patreon-) creates a non-capturing group that matches either prefix
$regex = "(?i)(?:youtube-|patreon-)$([regex]::Escape($Id))"
# Check if any file already exists that matches the video ID
$fileFound = $mp4Files | Where-Object { $_.Name -match $regex }
# If no file is found, notify the user
if ($fileFound) {
Write-Warning "File found with the YouTube ID: $Id"
}
Else {
# Check if this is a YouTube Short by getting video info (aspect ratio or duration)
<# Check if this is a YouTube Short by getting video info (aspect ratio or duration)
Write-Verbose -Message "Checking if video is a Short: $Title" -Verbose
$isShort = $false
$videoInfo = D:\yt-dlp\yt-dlp.exe -j $videoUrl
$videoInfoObj = $videoInfo | ConvertFrom-Json
if ($videoUrl -like "*shorts*") {
Write-Verbose "Identified as Shorts video because the URL contains 'shorts': $Id ##### $Title" -Verbose
$isShort = $true
@@ -188,7 +228,7 @@ ForEach ($obj in $json) {
# Determine if this is a short based on duration
Write-Verbose "Identified as Shorts video because the duration is 60 seconds or less: $Id ##### $Title" -Verbose
$isShort = $true
}
} <#
elseif ($videoInfoObj.width -and $videoInfoObj.height) {
# Calculate aspect ratio (height/width for vertical videos)
$aspectRatio = [double]$videoInfoObj.height / [double]$videoInfoObj.width
@@ -196,13 +236,13 @@ ForEach ($obj in $json) {
# Vertical video ( > 1.5:1)
$isShort = $true
}
}
}#>
# Set output path based on whether it's a short or not
if ($isShort) {
Write-Verbose -Message "Video detected as Short: $Title" -Verbose
# Note: This path must be determined before running the tracking check/download
$outputPath = Join-Path $folderPath "YouTube Shorts/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
$outputPath = Join-Path $folderPath "Unsorted/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
$shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
# Check if ID is in the tracking JSON file
@@ -215,7 +255,7 @@ ForEach ($obj in $json) {
}
else {
Write-Verbose -Message "Video detected as regular: $Title" -Verbose
$outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y,upload_date>%y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
$outputPath = Join-Path $folderPath "Unsorted/Season %(release_date>%y|Unknown)s/%(release_date>%m|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(extractor)s-%(id)s].%(ext)s"
}
# Execute yt-dlp command for non-rejected videos
@@ -248,7 +288,7 @@ ForEach ($obj in $json) {
}
if ($isShort -and $DownloadSuccessful) {
$shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
$shortDirPath = Split-Path $outputPath -Parent # Gets the directory path (e.g., \RegulationPodcast\YouTube Shorts\...)
Manage-ShortTracking -ShortDirPath $shortDirPath -VideoId $Id -IsNewDownload:$true
}
}