Merge c24c211105 into 6d4efe6523

Jellyfin.Server/Migrations/JellyfinMigrationService.cs

@@ -90,7 +90,7 @@ internal class JellyfinMigrationService
 
     private HashSet<MigrationStage> Migrations { get; set; }
 
-    public async Task CheckFirstTimeRunOrMigration(IApplicationPaths appPaths)
+    public async Task CheckFirstTimeRunAndMigration(IApplicationPaths appPaths)
     {
         var logger = _startupLogger.With(_loggerFactory.CreateLogger<JellyfinMigrationService>()).BeginGroup($"Migration Startup");
         logger.LogInformation("Initialise Migration service.");
@@ -130,55 +130,53 @@ internal class JellyfinMigrationService
 
             logger.LogInformation("Migration system initialisation completed.");
         }
-        else
+
+        // migrate any existing migration.xml files
+        var migrationConfigPath = Path.Join(appPaths.ConfigurationDirectoryPath, "migrations.xml");
+        var migrationOptions = File.Exists(migrationConfigPath)
+                ? (MigrationOptions)xmlSerializer.DeserializeFromFile(typeof(MigrationOptions), migrationConfigPath)!
+                : null;
+        if (migrationOptions != null && migrationOptions.Applied.Count > 0)
         {
-            // migrate any existing migration.xml files
-            var migrationConfigPath = Path.Join(appPaths.ConfigurationDirectoryPath, "migrations.xml");
-            var migrationOptions = File.Exists(migrationConfigPath)
-                 ? (MigrationOptions)xmlSerializer.DeserializeFromFile(typeof(MigrationOptions), migrationConfigPath)!
-                 : null;
-            if (migrationOptions != null && migrationOptions.Applied.Count > 0)
+            logger.LogInformation("Old migration style migration.xml detected. Migrate now.");
+            try
             {
-                logger.LogInformation("Old migration style migration.xml detected. Migrate now.");
-                try
+                var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false);
+                await using (dbContext.ConfigureAwait(false))
                 {
-                    var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false);
-                    await using (dbContext.ConfigureAwait(false))
+                    var historyRepository = dbContext.GetService<IHistoryRepository>();
+                    var appliedMigrations = await dbContext.Database.GetAppliedMigrationsAsync().ConfigureAwait(false);
+                    var lastOldAppliedMigration = Migrations
+                        .SelectMany(e => e.Where(e => e.Metadata.Key is not null)) // only consider migrations that have the key set as its the reference marker for legacy migrations.
+                        .Where(e => migrationOptions.Applied.Any(f => f.Id.Equals(e.Metadata.Key!.Value)))
+                        .Where(e => !appliedMigrations.Contains(e.BuildCodeMigrationId()))
+                        .OrderBy(e => e.BuildCodeMigrationId())
+                        .Last(); // this is the latest migration applied in the old migration.xml
+
+                    IReadOnlyList<CodeMigration> oldMigrations = [
+                        .. Migrations
+                        .SelectMany(e => e)
+                        .OrderBy(e => e.BuildCodeMigrationId())
+                        .TakeWhile(e => e.BuildCodeMigrationId() != lastOldAppliedMigration.BuildCodeMigrationId()),
+                        lastOldAppliedMigration
+                    ];
+                    // those are all migrations that had to run in the old migration system, even if not noted in the migration.xml file.
+
+                    var startupScripts = oldMigrations.Select(e => (Migration: e.Metadata, Script: historyRepository.GetInsertScript(new HistoryRow(e.BuildCodeMigrationId(), GetJellyfinVersion()))));
+                    foreach (var item in startupScripts)
                     {
-                        var historyRepository = dbContext.GetService<IHistoryRepository>();
-                        var appliedMigrations = await dbContext.Database.GetAppliedMigrationsAsync().ConfigureAwait(false);
-                        var lastOldAppliedMigration = Migrations
-                            .SelectMany(e => e.Where(e => e.Metadata.Key is not null)) // only consider migrations that have the key set as its the reference marker for legacy migrations.
-                            .Where(e => migrationOptions.Applied.Any(f => f.Id.Equals(e.Metadata.Key!.Value)))
-                            .Where(e => !appliedMigrations.Contains(e.BuildCodeMigrationId()))
-                            .OrderBy(e => e.BuildCodeMigrationId())
-                            .Last(); // this is the latest migration applied in the old migration.xml
-
-                        IReadOnlyList<CodeMigration> oldMigrations = [
-                            .. Migrations
-                            .SelectMany(e => e)
-                            .OrderBy(e => e.BuildCodeMigrationId())
-                            .TakeWhile(e => e.BuildCodeMigrationId() != lastOldAppliedMigration.BuildCodeMigrationId()),
-                            lastOldAppliedMigration
-                        ];
-                        // those are all migrations that had to run in the old migration system, even if not noted in the migration.xml file.
-
-                        var startupScripts = oldMigrations.Select(e => (Migration: e.Metadata, Script: historyRepository.GetInsertScript(new HistoryRow(e.BuildCodeMigrationId(), GetJellyfinVersion()))));
-                        foreach (var item in startupScripts)
-                        {
-                            logger.LogInformation("Migrate migration {Key}-{Name}.", item.Migration.Key, item.Migration.Name);
-                            await dbContext.Database.ExecuteSqlRawAsync(item.Script).ConfigureAwait(false);
-                        }
-
-                        logger.LogInformation("Rename old migration.xml to migration.xml.backup");
-                        File.Move(migrationConfigPath, Path.ChangeExtension(migrationConfigPath, ".xml.backup"), true);
+                        logger.LogInformation("Migrate migration {Key}-{Name}.", item.Migration.Key, item.Migration.Name);
+                        await dbContext.Database.ExecuteSqlRawAsync(item.Script).ConfigureAwait(false);
                     }
+
+                    logger.LogInformation("Rename old migration.xml to migration.xml.backup");
+                    File.Move(migrationConfigPath, Path.ChangeExtension(migrationConfigPath, ".xml.backup"), true);
                 }
-                catch (Exception ex)
-                {
-                    logger.LogCritical(ex, "Failed to apply migrations");
-                    throw;
-                }
+            }
+            catch (Exception ex)
+            {
+                logger.LogCritical(ex, "Failed to apply migrations");
+                throw;
             }
         }
     }

Jellyfin.Server/Migrations/Routines/MigrateLibraryDbCompatibilityCheck.cs

@@ -65,7 +65,7 @@ public class MigrateLibraryDbCompatibilityCheck : IAsyncMigrationRoutine
                 var result = cmd.ExecuteScalar()!;
                 if (!result.Equals(1L))
                 {
-                    throw new InvalidOperationException("Your database does not meet the required standard. Only upgrades from server version 10.9.11 or above are supported. Please upgrade first to server version 10.10.7 before attempting to upgrade afterwards to 10.11");
+                    throw new InvalidOperationException("Your database does not meet the required standard. Only upgrades from server version 10.9.11 or higher are supported. Please upgrade to server version 10.10.7 first, before attempting to upgrade to 10.11.");
                 }
             }
         }

Jellyfin.Server/Migrations/Routines/UpdateDatabaseToTenDotTen.cs

@@ -0,0 +1,187 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Emby.Server.Implementations.Data;
+using MediaBrowser.Controller;
+using Microsoft.Data.Sqlite;
+using Microsoft.Extensions.Logging;
+
+namespace Jellyfin.Server.Migrations.Routines;
+
+/// <summary>
+/// Updates the library.db to version 10.10.z in preperation for the EFCore migration.
+/// Replaces the migration code from the old SqliteItemRepository and SqliteUserDataRepository.
+/// </summary>
+[JellyfinMigration("2025-04-20T19:15:00", nameof(UpdateDatabaseToTenDotTen))]
+[JellyfinMigrationBackup(LegacyLibraryDb = true)]
+internal class UpdateDatabaseToTenDotTen : IDatabaseMigrationRoutine
+{
+    private const string DbFilename = "library.db";
+    private readonly ILogger<UpdateDatabaseToTenDotTen> _logger;
+    private readonly IServerApplicationPaths _paths;
+    private readonly List<(string TableName, string ColumnName, string Type)> _libraryDbTableColumns =
+    [
+        ("AncestorIds", "AncestorIdText", "Text"),
+
+        ("TypedBaseItems", "Path", "Text"),
+        ("TypedBaseItems", "StartDate", "DATETIME"),
+        ("TypedBaseItems", "EndDate", "DATETIME"),
+        ("TypedBaseItems", "ChannelId", "Text"),
+        ("TypedBaseItems", "IsMovie", "BIT"),
+        ("TypedBaseItems", "CommunityRating", "Float"),
+        ("TypedBaseItems", "CustomRating", "Text"),
+        ("TypedBaseItems", "IndexNumber", "INT"),
+        ("TypedBaseItems", "IsLocked", "BIT"),
+        ("TypedBaseItems", "Name", "Text"),
+        ("TypedBaseItems", "OfficialRating", "Text"),
+        ("TypedBaseItems", "MediaType", "Text"),
+        ("TypedBaseItems", "Overview", "Text"),
+        ("TypedBaseItems", "ParentIndexNumber", "INT"),
+        ("TypedBaseItems", "PremiereDate", "DATETIME"),
+        ("TypedBaseItems", "ProductionYear", "INT"),
+        ("TypedBaseItems", "ParentId", "GUID"),
+        ("TypedBaseItems", "Genres", "Text"),
+        ("TypedBaseItems", "SortName", "Text"),
+        ("TypedBaseItems", "ForcedSortName", "Text"),
+        ("TypedBaseItems", "RunTimeTicks", "BIGINT"),
+        ("TypedBaseItems", "DateCreated", "DATETIME"),
+        ("TypedBaseItems", "DateModified", "DATETIME"),
+        ("TypedBaseItems", "IsSeries", "BIT"),
+        ("TypedBaseItems", "EpisodeTitle", "Text"),
+        ("TypedBaseItems", "IsRepeat", "BIT"),
+        ("TypedBaseItems", "PreferredMetadataLanguage", "Text"),
+        ("TypedBaseItems", "PreferredMetadataCountryCode", "Text"),
+        ("TypedBaseItems", "DateLastRefreshed", "DATETIME"),
+        ("TypedBaseItems", "DateLastSaved", "DATETIME"),
+        ("TypedBaseItems", "IsInMixedFolder", "BIT"),
+        ("TypedBaseItems", "LockedFields", "Text"),
+        ("TypedBaseItems", "Studios", "Text"),
+        ("TypedBaseItems", "Audio", "Text"),
+        ("TypedBaseItems", "ExternalServiceId", "Text"),
+        ("TypedBaseItems", "Tags", "Text"),
+        ("TypedBaseItems", "IsFolder", "BIT"),
+        ("TypedBaseItems", "InheritedParentalRatingValue", "INT"),
+        ("TypedBaseItems", "UnratedType", "Text"),
+        ("TypedBaseItems", "TopParentId", "Text"),
+        ("TypedBaseItems", "TrailerTypes", "Text"),
+        ("TypedBaseItems", "CriticRating", "Float"),
+        ("TypedBaseItems", "CleanName", "Text"),
+        ("TypedBaseItems", "PresentationUniqueKey", "Text"),
+        ("TypedBaseItems", "OriginalTitle", "Text"),
+        ("TypedBaseItems", "PrimaryVersionId", "Text"),
+        ("TypedBaseItems", "DateLastMediaAdded", "DATETIME"),
+        ("TypedBaseItems", "Album", "Text"),
+        ("TypedBaseItems", "LUFS", "Float"),
+        ("TypedBaseItems", "NormalizationGain", "Float"),
+        ("TypedBaseItems", "IsVirtualItem", "BIT"),
+        ("TypedBaseItems", "SeriesName", "Text"),
+        ("TypedBaseItems", "UserDataKey", "Text"),
+        ("TypedBaseItems", "SeasonName", "Text"),
+        ("TypedBaseItems", "SeasonId", "GUID"),
+        ("TypedBaseItems", "SeriesId", "GUID"),
+        ("TypedBaseItems", "ExternalSeriesId", "Text"),
+        ("TypedBaseItems", "Tagline", "Text"),
+        ("TypedBaseItems", "ProviderIds", "Text"),
+        ("TypedBaseItems", "Images", "Text"),
+        ("TypedBaseItems", "ProductionLocations", "Text"),
+        ("TypedBaseItems", "ExtraIds", "Text"),
+        ("TypedBaseItems", "TotalBitrate", "INT"),
+        ("TypedBaseItems", "ExtraType", "Text"),
+        ("TypedBaseItems", "Artists", "Text"),
+        ("TypedBaseItems", "AlbumArtists", "Text"),
+        ("TypedBaseItems", "ExternalId", "Text"),
+        ("TypedBaseItems", "SeriesPresentationUniqueKey", "Text"),
+        ("TypedBaseItems", "ShowId", "Text"),
+        ("TypedBaseItems", "OwnerId", "Text"),
+        ("TypedBaseItems", "Width", "INT"),
+        ("TypedBaseItems", "Height", "INT"),
+        ("TypedBaseItems", "Size", "BIGINT"),
+
+        ("ItemValues", "CleanValue", "Text"),
+
+        ("Chapters2", "ImageDateModified", "DATETIME"),
+
+        ("mediastreams", "IsAvc", "BIT"),
+        ("mediastreams", "TimeBase", "TEXT"),
+        ("mediastreams", "CodecTimeBase", "TEXT"),
+        ("mediastreams", "Title", "TEXT"),
+        ("mediastreams", "NalLengthSize", "TEXT"),
+        ("mediastreams", "Comment", "TEXT"),
+        ("mediastreams", "CodecTag", "TEXT"),
+        ("mediastreams", "PixelFormat", "TEXT"),
+        ("mediastreams", "BitDepth", "INT"),
+        ("mediastreams", "RefFrames", "INT"),
+        ("mediastreams", "KeyFrames", "TEXT"),
+        ("mediastreams", "IsAnamorphic", "BIT"),
+        ("mediastreams", "ColorPrimaries", "TEXT"),
+        ("mediastreams", "ColorSpace", "TEXT"),
+        ("mediastreams", "ColorTransfer", "TEXT"),
+        ("mediastreams", "DvVersionMajor", "INT"),
+        ("mediastreams", "DvVersionMinor", "INT"),
+        ("mediastreams", "DvProfile", "INT"),
+        ("mediastreams", "DvLevel", "INT"),
+        ("mediastreams", "RpuPresentFlag", "INT"),
+        ("mediastreams", "ElPresentFlag", "INT"),
+        ("mediastreams", "BlPresentFlag", "INT"),
+        ("mediastreams", "DvBlSignalCompatibilityId", "INT"),
+        ("mediastreams", "IsHearingImpaired", "BIT"),
+        ("mediastreams", "Rotation", "INT")
+    ];
+
+    public UpdateDatabaseToTenDotTen(
+        ILogger<UpdateDatabaseToTenDotTen> logger,
+        IServerApplicationPaths paths)
+    {
+        _logger = logger;
+        _paths = paths;
+    }
+
+    /// <inheritdoc/>
+    public void Perform()
+    {
+        var dataPath = _paths.DataPath;
+
+        var dbPath = Path.Combine(dataPath, DbFilename);
+
+        _logger.LogInformation("Prepare database for EFCore migration: update to version 10.10.z");
+
+        // add missing columns
+        using var connection = new SqliteConnection($"Filename={dbPath}");
+        connection.Open();
+        using (var transaction = connection.BeginTransaction())
+        {
+            var existingColumns = GetExistingColumns(connection);
+
+            _libraryDbTableColumns
+                .Where(col => !existingColumns.Exists(exist => col.TableName == exist.TableName && col.ColumnName == exist.ColumnName))
+                .ToList()
+                .ForEach(col => connection.Execute($"ALTER TABLE {col.TableName} ADD COLUMN {col.ColumnName} {col.Type} NULL"));
+
+            transaction.Commit();
+        }
+
+        _logger.LogInformation("Database was successfully updated");
+    }
+
+    /// <summary>
+    /// Returns a list of all existing columns.
+    /// </summary>
+    private List<(string TableName, string ColumnName)> GetExistingColumns(SqliteConnection connection)
+    {
+        var existingColumns = new List<(string TableName, string ColumnName)>();
+        var existingColumnsQuery = @"SELECT t.name AS table_name, c.name AS column_name
+                                   FROM sqlite_master t
+                                   JOIN pragma_table_info(t.name) c
+                                   WHERE t.name IN ('AncestorIds', 'TypedBaseItems', 'ItemValues', 'Chapters2', 'mediastreams')";
+
+        foreach (var row in connection.Query(existingColumnsQuery))
+        {
+            if (row.TryGetString(0, out var tableName) && row.TryGetString(1, out var columnName))
+            {
+                existingColumns.Add((tableName, columnName));
+            }
+        }
+
+        return existingColumns;
+    }
+}

Jellyfin.Server/Program.cs

@@ -287,7 +287,7 @@ namespace Jellyfin.Server
             PrepareDatabaseProvider(startupService);
 
             var jellyfinMigrationService = ActivatorUtilities.CreateInstance<JellyfinMigrationService>(startupService);
-            await jellyfinMigrationService.CheckFirstTimeRunOrMigration(appPaths).ConfigureAwait(false);
+            await jellyfinMigrationService.CheckFirstTimeRunAndMigration(appPaths).ConfigureAwait(false);
             await jellyfinMigrationService.MigrateStepAsync(Migrations.Stages.JellyfinMigrationStageTypes.PreInitialisation, startupService).ConfigureAwait(false);
         }