From 0e8cf74fffcbd5083840da51965d39356d05c834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Ch=C3=A1vez=20Bornios?= Date: Sun, 27 Sep 2026 21:19:22 -0700 Subject: [PATCH 01/12] feat: add managed private plugin schemas Expose a versioned private-schema host, register plugin ownership in jellyfin_provider, and remove the JellyTrend-specific database contracts.\n\nThe host never exposes public, credentials, or database connections; it validates schema names, migrations, parameters, and read-only boundaries. --- Directory.Build.props | 6 +- .../Api/IPluginSchemaHost.cs | 79 ++ .../Api/PluginSchema.cs | 22 + .../Api/PluginSchemaMigration.cs | 14 + .../Api/PluginSchemaMigrationChecksum.cs | 16 + .../Api/PluginSchemaRequest.cs | 15 + .../Api/PluginSqlParameter.cs | 8 + .../Api/PostgresSchemaHostCapability.cs | 21 + .../Data/PostgresDatabaseProvider.cs | 14 + .../IJellyTrendStoreProvider.cs | 192 ---- .../IRecommendationQueryProvider.cs | 123 --- .../Logging/EfCoreDatabaseDiagnostics.cs | 42 + .../PostgresServiceRegistrator.cs | 10 +- .../Services/DbErrorLoggingInterceptor.cs | 70 ++ .../Services/ExportToSqliteService.cs | 8 +- .../Services/HomeQueryCacheInterceptor.cs | 480 ++++++++-- .../Services/JellyTrendPostgresStore.cs | 835 ------------------ .../Services/PostgresPluginSchemaHost.cs | 410 +++++++++ .../PostgresRecommendationQueryProvider.cs | 354 -------- README.en.md | 27 +- README.md | 27 +- .../HomeQueryCacheTests.cs | 54 +- .../JellyTrendContractTests.cs | 107 --- .../PluginSchemaHostTests.cs | 88 ++ .../ProviderContractTests.cs | 17 + build.yaml | 2 +- manifest.json | 2 +- 27 files changed, 1293 insertions(+), 1750 deletions(-) create mode 100644 Jellyfin.Database.Providers.Postgres/Api/IPluginSchemaHost.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Api/PluginSchema.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Api/PluginSchemaMigration.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Api/PluginSchemaMigrationChecksum.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Api/PluginSchemaRequest.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Api/PluginSqlParameter.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Api/PostgresSchemaHostCapability.cs delete mode 100644 Jellyfin.Database.Providers.Postgres/JellyTrendContract/IJellyTrendStoreProvider.cs delete mode 100644 Jellyfin.Database.Providers.Postgres/JellyTrendContract/IRecommendationQueryProvider.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Logging/EfCoreDatabaseDiagnostics.cs delete mode 100644 Jellyfin.Database.Providers.Postgres/Services/JellyTrendPostgresStore.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs delete mode 100644 Jellyfin.Database.Providers.Postgres/Services/PostgresRecommendationQueryProvider.cs delete mode 100644 Tests/Jellyfin.Database.Providers.Postgres.Tests/JellyTrendContractTests.cs create mode 100644 Tests/Jellyfin.Database.Providers.Postgres.Tests/PluginSchemaHostTests.cs diff --git a/Directory.Build.props b/Directory.Build.props index e2a3814..ae9e57d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 3.0.2.0 - 3.0.2.0 - 3.0.2.0 + 3.0.3.0 + 3.0.3.0 + 3.0.3.0 diff --git a/Jellyfin.Database.Providers.Postgres/Api/IPluginSchemaHost.cs b/Jellyfin.Database.Providers.Postgres/Api/IPluginSchemaHost.cs new file mode 100644 index 0000000..d47ecd6 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Api/IPluginSchemaHost.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Threading; +using System.Threading.Tasks; + +namespace Jellyfin.Database.Providers.Postgres.Api; + +/// +/// Provides isolated PostgreSQL storage for a plugin-owned schema. +/// +/// +/// This contract deliberately does not expose a connection string, an Npgsql connection, or Jellyfin's +/// public schema. Consumers must use Jellyfin's own contracts for Jellyfin data and use this host +/// only for state they own. Loaded plugins are trusted in-process code; this API is an isolation boundary +/// against accidental cross-plugin access, not a security boundary against malicious code. +/// +public interface IPluginSchemaHost +{ + /// + /// Opens a named private schema and binds it to its plugin owner. + /// + /// Schema name and immutable plugin owner. + /// Cancellation token. + /// An opaque handle for the plugin's schema. + Task OpenSchemaAsync(PluginSchemaRequest request, CancellationToken cancellationToken = default); + + /// + /// Opens the generated private schema assigned to , creating it when needed. + /// + /// The immutable identifier of the consuming plugin. + /// Cancellation token. + /// An opaque handle for the plugin's schema. + Task OpenSchemaAsync(Guid pluginId, CancellationToken cancellationToken = default); + + /// + /// Applies versioned DDL migrations in the schema represented by . + /// + /// Schema handle returned by . + /// Migrations to apply in their supplied order. + /// Cancellation token. + /// A task that completes after all migrations have been committed. + Task ApplyMigrationsAsync( + PluginSchema schema, + IReadOnlyList migrations, + CancellationToken cancellationToken = default); + + /// + /// Executes one parameterized data command inside the plugin's schema. + /// + /// Schema handle returned by . + /// A single unqualified SQL statement. + /// Parameters for . + /// Cancellation token. + /// The number of rows affected. + Task ExecuteAsync( + PluginSchema schema, + string sql, + IReadOnlyList? parameters = null, + CancellationToken cancellationToken = default); + + /// + /// Executes one parameterized query inside the plugin's schema and projects its rows without exposing + /// the underlying connection. + /// + /// Projection result type. + /// Schema handle returned by . + /// A single unqualified SQL query. + /// Synchronous projection of the open reader. + /// Parameters for . + /// Cancellation token. + /// The projected result. + Task QueryAsync( + PluginSchema schema, + string sql, + Func project, + IReadOnlyList? parameters = null, + CancellationToken cancellationToken = default); +} diff --git a/Jellyfin.Database.Providers.Postgres/Api/PluginSchema.cs b/Jellyfin.Database.Providers.Postgres/Api/PluginSchema.cs new file mode 100644 index 0000000..162224c --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Api/PluginSchema.cs @@ -0,0 +1,22 @@ +using System; + +namespace Jellyfin.Database.Providers.Postgres.Api; + +/// +/// Opaque reference to the private PostgreSQL schema assigned to a plugin. +/// +public sealed class PluginSchema +{ + internal PluginSchema(Guid pluginId, string name) + { + PluginId = pluginId; + Name = name; + } + + /// Gets the immutable identifier of the schema owner. + public Guid PluginId { get; } + + /// Gets the generated schema name for diagnostics only. + /// Consumers must not use this value to build SQL. + public string Name { get; } +} diff --git a/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaMigration.cs b/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaMigration.cs new file mode 100644 index 0000000..5227804 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaMigration.cs @@ -0,0 +1,14 @@ +using System; + +namespace Jellyfin.Database.Providers.Postgres.Api; + +/// +/// One immutable, plugin-owned schema migration. +/// +/// Stable migration identifier, unique within the plugin. +/// One unqualified DDL statement to execute. +public sealed record PluginSchemaMigration(string Id, string Sql) +{ + /// Gets a deterministic checksum used to detect modified applied migrations. + public string Checksum { get; } = PluginSchemaMigrationChecksum.Compute(Id, Sql); +} diff --git a/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaMigrationChecksum.cs b/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaMigrationChecksum.cs new file mode 100644 index 0000000..a721299 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaMigrationChecksum.cs @@ -0,0 +1,16 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace Jellyfin.Database.Providers.Postgres.Api; + +internal static class PluginSchemaMigrationChecksum +{ + internal static string Compute(string id, string sql) + { + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(sql); + var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(string.Concat(id, "\n", sql))); + return Convert.ToHexString(bytes); + } +} diff --git a/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaRequest.cs b/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaRequest.cs new file mode 100644 index 0000000..7049b71 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Api/PluginSchemaRequest.cs @@ -0,0 +1,15 @@ +using System; + +namespace Jellyfin.Database.Providers.Postgres.Api; + +/// +/// Identifies a plugin-owned PostgreSQL schema. +/// +/// Immutable identifier of the plugin that owns the schema. +/// Stable lowercase PostgreSQL schema name, for example jellytrend. +/// Whether an existing unclaimed schema may be bound to . +/// +/// Adoption exists only for a deliberate migration from a predecessor contract. Once claimed, the host +/// writes an ownership marker and rejects future requests by a different plugin identifier. +/// +public sealed record PluginSchemaRequest(Guid PluginId, string SchemaName, bool AdoptExistingSchema = false); diff --git a/Jellyfin.Database.Providers.Postgres/Api/PluginSqlParameter.cs b/Jellyfin.Database.Providers.Postgres/Api/PluginSqlParameter.cs new file mode 100644 index 0000000..f91371a --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Api/PluginSqlParameter.cs @@ -0,0 +1,8 @@ +namespace Jellyfin.Database.Providers.Postgres.Api; + +/// +/// A named value bound to a private-schema SQL statement. +/// +/// Parameter name without SQL syntax, for example itemId. +/// Value to bind. A null value is sent as . +public sealed record PluginSqlParameter(string Name, object? Value); diff --git a/Jellyfin.Database.Providers.Postgres/Api/PostgresSchemaHostCapability.cs b/Jellyfin.Database.Providers.Postgres/Api/PostgresSchemaHostCapability.cs new file mode 100644 index 0000000..964660b --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Api/PostgresSchemaHostCapability.cs @@ -0,0 +1,21 @@ +namespace Jellyfin.Database.Providers.Postgres.Api; + +/// +/// Public, dependency-free marker for discovering the private-schema host from another Jellyfin plugin. +/// +/// +/// A consumer obtains by asking Jellyfin's service provider for that +/// interface type from this loaded assembly. Consumers that cannot reference this assembly at compile time +/// may discover this marker and the interface by their full names through reflection. +/// +public static class PostgresSchemaHostCapability +{ + /// Stable protocol identifier used by reflection-based consumers. + public const string ProtocolName = "jellyfin.postgres.private-schema-host"; + + /// Current backwards-compatible protocol version. + public const int ProtocolVersion = 1; + + /// Full name of the service contract to request from Jellyfin DI. + public const string ContractTypeName = "Jellyfin.Database.Providers.Postgres.Api.IPluginSchemaHost"; +} diff --git a/Jellyfin.Database.Providers.Postgres/Data/PostgresDatabaseProvider.cs b/Jellyfin.Database.Providers.Postgres/Data/PostgresDatabaseProvider.cs index 8fd6606..04114f8 100644 --- a/Jellyfin.Database.Providers.Postgres/Data/PostgresDatabaseProvider.cs +++ b/Jellyfin.Database.Providers.Postgres/Data/PostgresDatabaseProvider.cs @@ -9,10 +9,12 @@ using System.Threading.Tasks; using Jellyfin.Database.Implementations; using Jellyfin.Database.Implementations.DbConfiguration; +using Jellyfin.Database.Providers.Postgres.Logging; using Jellyfin.Database.Providers.Postgres.Services; using Jellyfin.Database.Providers.Postgres.Services.Models; using MediaBrowser.Common.Configuration; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Npgsql; @@ -178,6 +180,13 @@ public void Initialise(DbContextOptionsBuilder options, DatabaseConfigurationOpt npgsql.CommandTimeout(commandTimeout); } }) + // Direct EF Core diagnostics are reliable even though Jellyfin's ILoggerFactory is constructed + // before plugins. Command lifecycle messages are intentionally excluded: the interceptor records + // slow commands and failures with richer, redacted SQL context in Postgres-*.log. + .LogTo( + EfCoreDatabaseDiagnostics.Write, + (eventId, level) => level >= LogLevel.Information + && !(eventId.Name?.Contains("Command", StringComparison.Ordinal) ?? false)) .AddInterceptors( new Jellyfin121MigrationInterceptor(), // This must precede HomeQueryCacheInterceptor: the cache buffers a miss by executing @@ -240,6 +249,11 @@ internal static NpgsqlConnectionStringBuilder BuildTunedConnectionString( { var builder = new NpgsqlConnectionStringBuilder(connectionString); + // Home/page prefetching opens an independent pooled connection after EF Core has returned the + // current page. Npgsql otherwise redacts Password from ConnectionString after Open(), leaving the + // background connection unable to authenticate. This value remains process-local and is never logged. + builder.PersistSecurityInfo = true; + // ShouldSerialize tells whether the key was explicitly present in the connection string. if (!builder.ShouldSerialize("Minimum Pool Size") && (config?.MinPoolSize ?? 0) > 0) { diff --git a/Jellyfin.Database.Providers.Postgres/JellyTrendContract/IJellyTrendStoreProvider.cs b/Jellyfin.Database.Providers.Postgres/JellyTrendContract/IJellyTrendStoreProvider.cs deleted file mode 100644 index 92d0cf1..0000000 --- a/Jellyfin.Database.Providers.Postgres/JellyTrendContract/IJellyTrendStoreProvider.cs +++ /dev/null @@ -1,192 +0,0 @@ -// ────────────────────────────────────────────────────────────────────────────── -// BUILD-TIME CONTRACT — do NOT modify. -// -// This file is a compile-time mirror of the public API declared in the -// JellyTrend plugin (Jellyfin.Plugin.JellyTrend.Api.IJellyTrendStoreProvider). -// It exists so this project can implement the interface without referencing the -// JellyTrend DLL at build time (no HintPath, no NuGet dependency). -// -// Rules: -// • Namespace, type names and method signatures MUST stay identical to the -// originals in JellyTrend — any drift will cause a MissingMethodException at -// runtime, because the call is forwarded by name and parameter type. -// • Every member uses only BCL types: a type declared in either assembly would -// never match the mirror on the other side. -// • When the JellyTrend contract changes, update this file to match. -// • This file ships in the Postgres plugin DLL but is NEVER loaded at runtime -// for its type identity — JellyTrend loads the real types from its own DLL. -// ────────────────────────────────────────────────────────────────────────────── -using System; - -namespace Jellyfin.Plugin.JellyTrend.Api; - -/// -/// Optional persistent store offered by a database provider plugin. -/// -/// -/// -/// When a provider implements this contract, JellyTrend keeps its data in that database instead of in -/// loose JSON files: the feature cache, the trending list and the per-user recommendations become rows -/// with a real history, so an update or a concurrent run cannot lose them and the next run only has to -/// process what changed. -/// -/// -/// Contract rules. Every member uses only BCL types on purpose: JellyTrend compiles its own copy -/// of the interface (same full name, different assembly), and the calls are forwarded by name and by -/// exact parameter type. A type declared in one of the two assemblies would never match. -/// -/// -/// Never throw for recoverable errors: return an empty array or instead, so -/// the caller can fall back to JSON without failing the task. The schema is created by the provider only -/// when is called — a server without JellyTrend installed never gets these -/// tables. -/// -/// -public interface IJellyTrendStoreProvider -{ - /// - /// Creates the schema and tables when they are missing. Idempotent. - /// - /// when the store is ready to be used. - bool EnsureSchema(); - - /// - /// Gets the version of the schema this provider manages. - /// - /// The schema version, or 0 when the schema does not exist yet. - int GetSchemaVersion(); - - /// - /// Describes the store for the log ("PostgreSQL 17.11, schema jellytrend"). - /// - /// A short description of the backend that holds the data. - string DescribeBackend(); - - /// - /// Replaces the cached features of the given items. - /// - /// Item ids, one per entry. - /// Features of each item as a JSON object, aligned with . - /// Fingerprint of the item metadata, used to skip unchanged items. - /// The number of rows written. - int ReplaceItemFeatures(Guid[] itemIds, string[] facetsJson, string[] contentHashes); - - /// - /// Reads the cached features of the given items. - /// - /// Item ids to read. - /// One JSON object per id, in the same order; null for ids with no cached features. - string?[] GetItemFeatures(Guid[] itemIds); - - /// - /// Reads every cached feature document at once. - /// - /// A JSON object keyed by item id, or null when there is nothing cached. - string? GetItemFeaturesJson(); - - /// - /// Replaces the whole trending list. - /// - /// Local item ids of the trending entries. - /// One trending entry per id, as JSON, aligned with . - /// Moment the list was refreshed from the external provider. - void ReplaceTrending(Guid[] itemIds, string[] entriesJson, DateTime syncedAt); - - /// - /// Reads the trending list as the JSON document JellyTrend already understands. - /// - /// The document, or null when there is nothing stored. - string? GetTrendingJson(); - - /// - /// Replaces the stored recommendations of a user. - /// - /// User the recommendations belong to. - /// Recommended item ids, best first. - /// Score of each item, aligned with . - /// Moment the run produced them. - /// Run that produced them, for the history. - void ReplaceUserRecommendations(Guid userId, Guid[] itemIds, double[] scores, DateTime generatedAt, Guid runId); - - /// - /// Reads the stored recommendations of a user. - /// - /// User to read. - /// The recommended item ids, best first; empty when there is nothing stored. - Guid[] GetUserRecommendations(Guid userId); - - /// - /// Lists the users that have stored recommendations. - /// - /// User ids, newest run first. - Guid[] GetUsersWithRecommendations(); - - /// - /// Replaces the items a user must not be shown again (already seen, dismissed). - /// - /// User the suppression belongs to. - /// Suppressed item ids. - /// Reason of each item, aligned with . - void ReplaceSuppressed(Guid userId, Guid[] itemIds, string[] reasons); - - /// - /// Reads the items a user must not be shown again. - /// - /// User to read. - /// The suppressed item ids; empty when there are none. - Guid[] GetSuppressed(Guid userId); - - /// - /// Replaces the aggregated consumption of a user's titles. - /// - /// User the consumption belongs to. - /// Titles the user has interacted with. - /// One consumption document per id, aligned with . - /// The number of rows written. - int ReplaceConsumption(Guid userId, Guid[] itemIds, string[] consumptionJson); - - /// - /// Reads every aggregated consumption document of a user. - /// - /// User to read. - /// A JSON object keyed by item id, or null when there is nothing stored. - string? GetUserConsumption(Guid userId); - - /// - /// Replaces the learned taste profile of a user. - /// - /// User the profile belongs to. - /// Facet of each affinity. - /// Value of each affinity. - /// Second facet of a combined affinity, empty when it is a single value. - /// Second value of a combined affinity, empty when it is a single value. - /// Weight of each affinity, aligned with . - /// The number of rows written. - int ReplaceAffinities(Guid userId, string[] facets, string[] values, string[] pairedFacets, string[] pairedValues, double[] weights); - - /// - /// Reads the learned taste profile of a user. - /// - /// User to read. - /// A JSON array of affinities, or null when there is nothing stored. - string? GetAffinities(Guid userId); - - /// - /// Records the start of a run. - /// - /// Kind of run ("recommendations" or "trending"). - /// Moment the run started. - /// The run id, to pass to . - Guid StartRun(string kind, DateTime startedAt); - - /// - /// Records the outcome of a run. - /// - /// Run to close. - /// Outcome ("ok", "partial", "failed" or "canceled"). - /// Users that finished without errors. - /// Users that failed. - /// Wall-clock duration in milliseconds. - /// Optional detail for the history. - void FinishRun(Guid runId, string status, int usersOk, int usersFailed, int durationMs, string? message); -} diff --git a/Jellyfin.Database.Providers.Postgres/JellyTrendContract/IRecommendationQueryProvider.cs b/Jellyfin.Database.Providers.Postgres/JellyTrendContract/IRecommendationQueryProvider.cs deleted file mode 100644 index 1aa7e34..0000000 --- a/Jellyfin.Database.Providers.Postgres/JellyTrendContract/IRecommendationQueryProvider.cs +++ /dev/null @@ -1,123 +0,0 @@ -// ────────────────────────────────────────────────────────────────────────────── -// BUILD-TIME CONTRACT — do NOT modify. -// -// This file is a compile-time mirror of the public API declared in the -// JellyTrend plugin (Jellyfin.Plugin.JellyTrend.Api.IRecommendationQueryProvider). -// It exists so this project can implement the interface without referencing the -// JellyTrend DLL at build time (no HintPath, no NuGet dependency). -// -// Rules: -// • Namespace, type names and method signatures MUST stay identical to the -// originals in JellyTrend — any drift will cause a TypeLoadException at runtime. -// • When the JellyTrend contract changes, update this file to match. -// • This file ships in the Postgres plugin DLL but is NEVER loaded at runtime -// for its type identity — Jellyfin loads the real types from the JellyTrend DLL. -// ────────────────────────────────────────────────────────────────────────────── -using System; -using System.Collections.Generic; - -namespace Jellyfin.Plugin.JellyTrend.Api; - -/// -/// Optional database-provider-specific query backend for the recommendation engine. -/// When registered in the Jellyfin DI container (e.g. by the PostgreSQL Database Provider -/// plugin), the recommendation engine delegates its library queries to this interface -/// instead of using ILibraryManager.GetItemList. -/// -/// -/// Implementations must never throw for recoverable errors — return an empty list instead. -/// If the provider is not registered, the engine falls back to ILibraryManager -/// transparently so JellyTrend works on SQLite without changes. -/// -public interface IRecommendationQueryProvider -{ - /// - /// Returns movies the user has played, up to items. - /// - /// The user whose play history to query. - /// Maximum number of items to return. - /// Lightweight projections of played movies. - IReadOnlyList GetPlayedMovies(Guid userId, int limit); - - /// - /// Returns movies the user has started but not finished (resume candidates). - /// - /// The user whose resume queue to query. - /// Maximum number of items to return. - /// Lightweight projections of in-progress movies. - IReadOnlyList GetResumableMovies(Guid userId, int limit); - - /// - /// Returns unwatched movie candidates that match the given genres. - /// - /// User whose played status is checked. - /// Genres to filter by (OR logic — any match qualifies). - /// Library root ids to scope the query. - /// Maximum number of items to return. - /// Lightweight projections of matching unwatched movies. - IReadOnlyList GetUnwatchedMoviesByGenres( - Guid userId, - IReadOnlyList genres, - IReadOnlyList topParentIds, - int limit); - - /// - /// Returns unwatched movie candidates associated with the given person ids. - /// - /// User whose played status is checked. - /// Person ids to filter by (OR logic). - /// Library root ids to scope the query. - /// Maximum number of items to return. - /// Lightweight projections of matching unwatched movies. - IReadOnlyList GetUnwatchedMoviesByPersons( - Guid userId, - IReadOnlyList personIds, - IReadOnlyList topParentIds, - int limit); - - /// - /// Returns unwatched movie candidates that carry the given tags. - /// - /// User whose played status is checked. - /// Tags to filter by (OR logic). - /// Library root ids to scope the query. - /// Maximum number of items to return. - /// Lightweight projections of matching unwatched movies. - IReadOnlyList GetUnwatchedMoviesByTags( - Guid userId, - IReadOnlyList tags, - IReadOnlyList topParentIds, - int limit); - - /// - /// Returns a random sample of unwatched movies for cold-start users. - /// - /// User whose played status is checked. - /// Library root ids to scope the query. - /// Maximum number of items to return. - /// Lightweight projections of random unwatched movies. - IReadOnlyList GetRandomUnwatchedMovies( - Guid userId, - IReadOnlyList topParentIds, - int limit); -} - -/// -/// Minimal projection of a library item used by . -/// Contains only the fields needed by the recommendation engine for scoring. -/// -/// Jellyfin item GUID. -/// TMDB provider id, or null if not available. -/// Genre names. -/// Tag names. -/// Studio names. -/// Community rating (0–10), or null. -/// Premiere date in UTC, or null. -public sealed record RecommendationItem( - Guid Id, - string? TmdbId, - IReadOnlyList Genres, - IReadOnlyList Tags, - IReadOnlyList Studios, - float? CommunityRating, - DateTime? PremiereDate); diff --git a/Jellyfin.Database.Providers.Postgres/Logging/EfCoreDatabaseDiagnostics.cs b/Jellyfin.Database.Providers.Postgres/Logging/EfCoreDatabaseDiagnostics.cs new file mode 100644 index 0000000..55341fb --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Logging/EfCoreDatabaseDiagnostics.cs @@ -0,0 +1,42 @@ +using System; +using System.Text.RegularExpressions; + +namespace Jellyfin.Database.Providers.Postgres.Logging; + +/// +/// Routes EF Core lifecycle, warning and error diagnostics into the PostgreSQL provider log. +/// +internal static class EfCoreDatabaseDiagnostics +{ + private const int MaxMessageLength = 2000; + + private static readonly Regex SecretRegex = new( + @"(?i)(password|token|access[ _-]?key)\s*=\s*[^;\s,]+", + RegexOptions.CultureInvariant, + TimeSpan.FromMilliseconds(100)); + + /// Writes one EF Core diagnostic without exposing connection secrets. + /// Message emitted by EF Core. + public static void Write(string message) + { + var safe = SecretRegex.Replace(message, "$1=*****"); + if (safe.Length > MaxMessageLength) + { + safe = string.Concat(safe.AsSpan(0, MaxMessageLength), "…"); + } + + safe = string.Concat("[EF] ", safe); + if (safe.Contains("fail:", StringComparison.OrdinalIgnoreCase)) + { + PostgresLog.Error(safe); + } + else if (safe.Contains("warn:", StringComparison.OrdinalIgnoreCase)) + { + PostgresLog.Warn(safe); + } + else + { + PostgresLog.Info(safe); + } + } +} diff --git a/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs b/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs index 11a5985..2e66b38 100644 --- a/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs +++ b/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs @@ -1,4 +1,5 @@ using System; +using Jellyfin.Database.Providers.Postgres.Api; using Jellyfin.Database.Providers.Postgres.Services; using Jellyfin.Database.Providers.Postgres.Tasks; using MediaBrowser.Controller; @@ -19,6 +20,9 @@ public class PostgresServiceRegistrator : IPluginServiceRegistrator /// public void RegisterServices(IServiceCollection serviceCollection, IServerApplicationHost applicationHost) { + // This is intentionally the only public cross-plugin database contract. It gives a consumer an + // isolated schema but never a connection, credential, or route to Jellyfin's public schema. + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); @@ -29,11 +33,5 @@ public void RegisterServices(IServiceCollection serviceCollection, IServerApplic serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); - - // No JellyTrend registration happens here on purpose. This assembly carries a mirror of the - // JellyTrend contract, which is the same full name in a different assembly: registering it would - // hand JellyTrend an object that does not implement ITS interface, and resolving it throws - // InvalidCastException. The plugin that needs the data looks for it, so JellyTrend discovers - // PostgresRecommendationQueryProvider by reflection over the loaded plugins. } } diff --git a/Jellyfin.Database.Providers.Postgres/Services/DbErrorLoggingInterceptor.cs b/Jellyfin.Database.Providers.Postgres/Services/DbErrorLoggingInterceptor.cs index f837716..0eb68af 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/DbErrorLoggingInterceptor.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/DbErrorLoggingInterceptor.cs @@ -23,6 +23,10 @@ namespace Jellyfin.Database.Providers.Postgres.Services; /// public sealed class DbErrorLoggingInterceptor : DbCommandInterceptor { + // A command above this threshold is useful operational information, while logging every successful + // command would turn the dedicated database log into an I/O bottleneck during Home fan-out. + private const int SlowCommandThresholdMilliseconds = 500; + // Longest SQL preview written for a failed command (EF Core statements are long, and the head of // the statement is what identifies it). private const int MaxSqlPreviewLength = 500; @@ -60,6 +64,60 @@ public override Task CommandFailedAsync( return Task.CompletedTask; } + /// + public override DbDataReader ReaderExecuted(DbCommand command, CommandExecutedEventData eventData, DbDataReader result) + { + LogSlowCommand(command, eventData.Duration, "reader"); + return result; + } + + /// + public override ValueTask ReaderExecutedAsync( + DbCommand command, + CommandExecutedEventData eventData, + DbDataReader result, + CancellationToken cancellationToken = default) + { + LogSlowCommand(command, eventData.Duration, "reader"); + return ValueTask.FromResult(result); + } + + /// + public override int NonQueryExecuted(DbCommand command, CommandExecutedEventData eventData, int result) + { + LogSlowCommand(command, eventData.Duration, "non-query"); + return result; + } + + /// + public override ValueTask NonQueryExecutedAsync( + DbCommand command, + CommandExecutedEventData eventData, + int result, + CancellationToken cancellationToken = default) + { + LogSlowCommand(command, eventData.Duration, "non-query"); + return ValueTask.FromResult(result); + } + + /// + public override object? ScalarExecuted(DbCommand command, CommandExecutedEventData eventData, object? result) + { + LogSlowCommand(command, eventData.Duration, "scalar"); + return result; + } + + /// + public override ValueTask ScalarExecutedAsync( + DbCommand command, + CommandExecutedEventData eventData, + object? result, + CancellationToken cancellationToken = default) + { + LogSlowCommand(command, eventData.Duration, "scalar"); + return ValueTask.FromResult(result); + } + // ── Helpers ─────────────────────────────────────────────────────────────── private static void LogCommandError(DbCommand command, Exception? exception) @@ -106,6 +164,18 @@ private static void LogCommandError(DbCommand command, Exception? exception) } } + private static void LogSlowCommand(DbCommand command, TimeSpan duration, string commandKind) + { + if (duration.TotalMilliseconds < SlowCommandThresholdMilliseconds) + { + return; + } + + PostgresLog.Warn(string.Create( + CultureInfo.InvariantCulture, + $"[DB] Consulta lenta ({commandKind}): {duration.TotalMilliseconds:F0} ms | SQL: {Truncate(command.CommandText, MaxSqlPreviewLength)}")); + } + /// /// Builds the context line of a failed command: where it came from, the statement and, when the /// administrator enabled it, the parameter values that actually caused the failure. diff --git a/Jellyfin.Database.Providers.Postgres/Services/ExportToSqliteService.cs b/Jellyfin.Database.Providers.Postgres/Services/ExportToSqliteService.cs index 3342dc0..ddd21e6 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/ExportToSqliteService.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/ExportToSqliteService.cs @@ -30,12 +30,6 @@ public sealed class ExportToSqliteService : IDisposable private static readonly HashSet SkipTables = new(StringComparer.OrdinalIgnoreCase) { "__EFMigrationsHistory", "__EFMigrationsLock" }; - // Tablas propias del plugin JellyTrend (esquema jellytrend). La garantia real es el esquema: la - // exportacion solo enumera 'public', asi que no las ve. Esta lista es la segunda red, para que ni - // apareciendo en 'public' viajen a un SQLite donde Jellyfin las abriria sin sus datos. - private static readonly HashSet JellyTrendTables = new(StringComparer.OrdinalIgnoreCase) - { "item_features", "trending_item", "user_recommendation", "user_suppression", "sync_run", "schema_version" }; - // Jellyfin's native SQLite stores GUIDs in UPPERCASE. PostgreSQL uses lowercase. // This pattern detects UUID strings so we can normalise them on export. private static readonly System.Text.RegularExpressions.Regex GuidPattern = @@ -181,7 +175,7 @@ private async Task RunExportAsync(string pgConnStr, string sqliteDbPath, Cancell await pgConn.OpenAsync(ct).ConfigureAwait(false); var tables = await ExportHelpers.GetUserTablesAsync(pgConn, ct).ConfigureAwait(false); - tables = tables.Where(t => !SkipTables.Contains(t) && !JellyTrendTables.Contains(t)).ToList(); + tables = tables.Where(t => !SkipTables.Contains(t)).ToList(); Log($"Tablas a exportar: {tables.Count}"); PostgresLog.Info($"[Export] Tablas a exportar: {tables.Count}"); diff --git a/Jellyfin.Database.Providers.Postgres/Services/HomeQueryCacheInterceptor.cs b/Jellyfin.Database.Providers.Postgres/Services/HomeQueryCacheInterceptor.cs index 3376d64..155b84c 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/HomeQueryCacheInterceptor.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/HomeQueryCacheInterceptor.cs @@ -10,9 +10,11 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Jellyfin.Database.Providers.Postgres.Logging; using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; +using Npgsql; namespace Jellyfin.Database.Providers.Postgres.Services; @@ -44,6 +46,9 @@ public sealed class HomeQueryCacheInterceptor : DbCommandInterceptor // Result sets larger than this are still buffered (see ReaderExecuting) but not retained // in the cache, so a big query cannot pin a large DataTable in memory. private const int MaxCacheableRows = 1000; + private const int MaxCacheableCells = 10000; + private const int CacheSizeLimit = 50000; + private const int MaxPrefetchPageSize = 100; /// /// Marker a query can add with TagWith to stay out of the cache entirely, for statements whose @@ -52,6 +57,12 @@ public sealed class HomeQueryCacheInterceptor : DbCommandInterceptor internal const string NoCacheMarker = "__jellyfin_no_cache"; private static readonly TimeSpan CacheTtl = TimeSpan.FromSeconds(30); + private static readonly TimeSpan PrefetchTimeout = TimeSpan.FromSeconds(2); + + private static readonly Regex PagedBaseItemQueryRegex = new( + @"\bLIMIT\s+(?@[A-Za-z0-9_]+|\d+)\s+OFFSET\s+(?@[A-Za-z0-9_]+|\d+)", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, + TimeSpan.FromMilliseconds(100)); // Only intercept queries on the tables that Jellyfin fans out on home-page load. The order is fixed // because it participates in the cache key. @@ -69,7 +80,9 @@ public sealed class HomeQueryCacheInterceptor : DbCommandInterceptor // kept serving rows inserted by a transaction that had already been rolled back. private static readonly MemoryCache Cache = new(new MemoryCacheOptions { - SizeLimit = 200, + // A unit is one materialised cell. This bounds memory while retaining enough + // compact home rows and 100-item pages to make repeat navigation immediate. + SizeLimit = CacheSizeLimit, }); // One counter per cached table, bumped by every write to it. @@ -80,6 +93,13 @@ public sealed class HomeQueryCacheInterceptor : DbCommandInterceptor // cache keys for the lifetime of the server. private static readonly ConcurrentDictionary InFlight = new(StringComparer.Ordinal); + // Speculative work is deliberately capped. A real Jellyfin request must never wait for a page warm-up. + private static readonly SemaphoreSlim PrefetchSlot = new(1, 1); + private static readonly ConcurrentDictionary PrefetchInFlight = new(StringComparer.Ordinal); + + private static int _prefetchCompletedCount; + private static int _prefetchedPageHitCount; + private readonly ILogger? _logger; /// @@ -95,6 +115,15 @@ public HomeQueryCacheInterceptor(ILogger? logger = null) /// Exposed internally only for regression tests; it must return to zero after every fill. internal static int InFlightCount => InFlight.Count; + /// Gets the number of page prefetches currently queued or executing. + internal static int PrefetchInFlightCount => PrefetchInFlight.Count; + + /// Gets completed prefetched pages since the last purge. + internal static int PrefetchCompletedCount => Volatile.Read(ref _prefetchCompletedCount); + + /// Gets requests answered from a prefetched page since the last purge. + internal static int PrefetchedPageHitCount => Volatile.Read(ref _prefetchedPageHitCount); + /// /// Empties the shared cache. Needed after an operation that changes data behind EF's back, such as a /// database restore, because no write command of this process is seen by the interceptor. @@ -102,6 +131,8 @@ public HomeQueryCacheInterceptor(ILogger? logger = null) internal static void Purge() { Cache.Compact(1.0); + Interlocked.Exchange(ref _prefetchCompletedCount, 0); + Interlocked.Exchange(ref _prefetchedPageHitCount, 0); } /// @@ -152,10 +183,9 @@ public override async ValueTask> ReaderExecutin } var cacheKey = CanUseCache(command) ? BuildCacheKey(command) : null; - if (cacheKey is not null - && Cache.TryGetValue(cacheKey, out DataTable? cached) - && cached is not null) + if (cacheKey is not null && TryGetCachedResult(cacheKey, out var cached)) { + ScheduleNextPagePrefetch(command, cached); if (_logger?.IsEnabled(LogLevel.Debug) == true) { _logger?.LogDebug( @@ -216,10 +246,9 @@ public override InterceptionResult ReaderExecuting( // Same as the async path: a random query may be buffered but must never be cached. var cacheKey = CanUseCache(command) && !HasOrderByRandom(command) ? BuildCacheKey(command) : null; - if (cacheKey is not null - && Cache.TryGetValue(cacheKey, out DataTable? cached) - && cached is not null) + if (cacheKey is not null && TryGetCachedResult(cacheKey, out var cached)) { + ScheduleNextPagePrefetch(command, cached); return InterceptionResult.SuppressWithResult(new DataTableReader(cached)); } @@ -238,23 +267,13 @@ private static bool IsCacheable(DbCommand command) return false; } - // Only SELECT statements var firstWord = FirstToken(sql); if (firstWord is null || !firstWord.Equals("SELECT", StringComparison.OrdinalIgnoreCase)) { return false; } - // Must reference at least one cacheable table - foreach (var table in CacheableTables) - { - if (sql.Contains(table, StringComparison.Ordinal)) - { - return true; - } - } - - return false; + return CacheableTables.Any(table => sql.Contains(table, StringComparison.Ordinal)); } private static string? FirstToken(string sql) @@ -366,8 +385,9 @@ private async ValueTask> ExecuteAndCacheAsync( try { // A concurrent request may have completed the same fill while this request was waiting. - if (Cache.TryGetValue(cacheKey, out DataTable? cached) && cached is not null) + if (TryGetCachedResult(cacheKey, out var cached)) { + ScheduleNextPagePrefetch(command, cached); return InterceptionResult.SuppressWithResult(new DataTableReader(cached)); } @@ -397,35 +417,25 @@ private async ValueTask> ExecuteAndCacheCoreAsy await command.Connection.OpenAsync(ct).ConfigureAwait(false); } - var dt = new DataTable(); var readerObj = await command.ExecuteReaderAsync(ct).ConfigureAwait(false); await using (readerObj.ConfigureAwait(false)) { - dt.Load(readerObj); - } + var dt = await MaterializeTypedAsync(readerObj, ct).ConfigureAwait(false); - if (cacheKey is not null) - { - CacheResult(cacheKey, dt); - } + if (cacheKey is not null) + { + CacheResult(cacheKey, dt, command, isPrefetch: false); + ScheduleNextPagePrefetch(command, dt); + } - if (_logger?.IsEnabled(LogLevel.Debug) == true) - { - _logger?.LogDebug( - "HomeQueryCache {State} ({Cols} cols x {Rows} rows): {Sql}", - cacheKey is null ? "BUFFER ONLY" : "MISS (cached)", - dt.Columns.Count, - dt.Rows.Count, - command.CommandText[..Math.Min(command.CommandText.Length, 120)]); + return InterceptionResult.SuppressWithResult(new DataTableReader(dt)); } - - return InterceptionResult.SuppressWithResult(new DataTableReader(dt)); } catch (Exception ex) { // Never break Jellyfin — if caching fails, let EF Core execute normally. // Return the original result so the pipeline continues without interception. - _logger?.LogWarning(ex, "HomeQueryCache execution failed — falling back to direct DB query."); + PostgresLog.Warn("[Cache] Ejecución async falló; se usará PostgreSQL directo.", ex); return default; // default(InterceptionResult) = no suppression } } @@ -445,8 +455,9 @@ private InterceptionResult ExecuteAndCache(DbCommand command, stri gate.Wait(); try { - if (Cache.TryGetValue(cacheKey, out DataTable? cached) && cached is not null) + if (TryGetCachedResult(cacheKey, out var cached)) { + ScheduleNextPagePrefetch(command, cached); return InterceptionResult.SuppressWithResult(new DataTableReader(cached)); } @@ -469,23 +480,23 @@ private InterceptionResult ExecuteAndCacheCore(DbCommand command, command.Connection.Open(); } - var dt = new DataTable(); using (var reader = command.ExecuteReader()) { - dt.Load(reader); - } + var dt = MaterializeTyped(reader); - if (cacheKey is not null) - { - CacheResult(cacheKey, dt); - } + if (cacheKey is not null) + { + CacheResult(cacheKey, dt, command, isPrefetch: false); + ScheduleNextPagePrefetch(command, dt); + } - return InterceptionResult.SuppressWithResult(new DataTableReader(dt)); + return InterceptionResult.SuppressWithResult(new DataTableReader(dt)); + } } catch (Exception ex) { // Never break Jellyfin — if buffering fails, let EF Core execute normally. - _logger?.LogWarning(ex, "HomeQueryCache execution failed — falling back to direct DB query."); + PostgresLog.Warn("[Cache] Ejecución sync falló; se usará PostgreSQL directo.", ex); return default; } } @@ -495,21 +506,228 @@ private InterceptionResult ExecuteAndCacheCore(DbCommand command, /// /// Cache key derived from SQL and parameters. /// Materialised result set. - private void CacheResult(string cacheKey, DataTable dt) + /// The EF Core command that produced the result. + /// Whether this result was warmed before a client requested it. + private void CacheResult(string cacheKey, DataTable dt, DbCommand command, bool isPrefetch) { - if (dt.Rows.Count > MaxCacheableRows) + var cacheSize = GetCacheSize(dt); + if (dt.Rows.Count > MaxCacheableRows || cacheSize > MaxCacheableCells) { // Big result sets are buffered for correctness but not retained. return; } - Cache.Set(cacheKey, dt, new MemoryCacheEntryOptions + Cache.Set(cacheKey, new CachedResult(dt, isPrefetch), new MemoryCacheEntryOptions { AbsoluteExpirationRelativeToNow = CacheTtl, - Size = dt.Columns.Count, // proxy: more columns → heavier entry + Priority = IsHomeFanOutQuery(command) ? CacheItemPriority.High : CacheItemPriority.Normal, + Size = cacheSize, }); } + private static bool TryGetCachedResult(string cacheKey, out DataTable result) + { + if (Cache.TryGetValue(cacheKey, out CachedResult? cached) && cached is not null) + { + if (cached.IsPrefetched) + { + Interlocked.Increment(ref _prefetchedPageHitCount); + } + + result = cached.Table; + return true; + } + + result = null!; + return false; + } + + private static int GetCacheSize(DataTable table) + => Math.Max(1, checked(table.Rows.Count * Math.Max(1, table.Columns.Count))); + + // Home fans out into several small, repeatable BaseItems queries. Retaining those before ordinary + // library pages makes a refresh or a second client opening Home a memory-only operation. + private static bool IsHomeFanOutQuery(DbCommand command) + => command.CommandText.Contains("\"BaseItems\"", StringComparison.Ordinal) + && command.CommandText.Contains("LIMIT", StringComparison.OrdinalIgnoreCase) + && !command.CommandText.Contains("OFFSET", StringComparison.OrdinalIgnoreCase); + + private void ScheduleNextPagePrefetch(DbCommand command, DataTable currentPage) + { + if (!TryCreatePagePrefetchPlan(command, out var plan) + || currentPage.Rows.Count != plan.Limit) + { + return; + } + + var snapshot = CommandSnapshot.Create(command, plan); + if (snapshot is null || !PrefetchInFlight.TryAdd(snapshot.CacheKey, 0)) + { + return; + } + + _ = Task.Run(() => PrefetchNextPageAsync(snapshot)); + } + + private async Task PrefetchNextPageAsync(CommandSnapshot snapshot) + { + // Do not queue speculative work: if the server is active, the following real request will simply + // execute normally. This is intentionally a best-effort optimisation. + if (!PrefetchSlot.Wait(0)) + { + PrefetchInFlight.TryRemove(snapshot.CacheKey, out _); + return; + } + + try + { + if (Cache.TryGetValue(snapshot.CacheKey, out CachedResult? existing) && existing is not null) + { + return; + } + + using var timeout = new CancellationTokenSource(PrefetchTimeout); + using var connection = new NpgsqlConnection(snapshot.ConnectionString); + await connection.OpenAsync(timeout.Token).ConfigureAwait(false); + using var command = connection.CreateCommand(); + SetGeneratedCommandText(command, snapshot.CommandText); + foreach (var parameter in snapshot.Parameters) + { + command.Parameters.Add(parameter.Create(command)); + } + + using var reader = await command.ExecuteReaderAsync(timeout.Token).ConfigureAwait(false); + var page = await MaterializeTypedAsync(reader, timeout.Token).ConfigureAwait(false); + CacheResult(snapshot.CacheKey, page, command, isPrefetch: true); + Interlocked.Increment(ref _prefetchCompletedCount); + + if (_logger?.IsEnabled(LogLevel.Debug) == true) + { + _logger.LogDebug("HomeQueryCache prefetched page at offset {Offset} ({Rows} rows).", snapshot.Offset, page.Rows.Count); + } + } + catch (OperationCanceledException) + { + // A prefetch that cannot finish quickly is intentionally discarded. + } + catch (Exception ex) + { + PostgresLog.Warn("[Cache] La precarga de la siguiente página falló; la siguiente solicitud consultará PostgreSQL.", ex); + } + finally + { + PrefetchSlot.Release(); + PrefetchInFlight.TryRemove(snapshot.CacheKey, out _); + } + } + + private static bool TryCreatePagePrefetchPlan(DbCommand command, out PagePrefetchPlan plan) + { + plan = default; + var sql = command.CommandText; + if (!CanUseCache(command) + || HasOrderByRandom(command) + || !sql.Contains("\"BaseItems\"", StringComparison.Ordinal) + || !sql.Contains("ORDER BY", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var matches = PagedBaseItemQueryRegex.Matches(sql); + if (matches.Count != 1) + { + // A query with more than one LIMIT/OFFSET pair is normally a navigation include. It cannot be + // safely advanced by changing one offset because that would alter a different result set. + return false; + } + + var match = matches[0]; + + if (!TryGetPagingValue(command, match.Groups["limit"].Value, out var limit) + || !TryGetPagingValue(command, match.Groups["offset"].Value, out var offset) + || limit is <= 0 or > MaxPrefetchPageSize + || offset < 0 + || offset > long.MaxValue - limit) + { + return false; + } + + plan = new PagePrefetchPlan(match.Groups["offset"].Value, checked(offset + limit), checked((int)limit)); + return true; + } + + private static bool TryGetPagingValue(DbCommand command, string token, out long value) + { + value = 0; + if (long.TryParse(token, NumberStyles.None, CultureInfo.InvariantCulture, out value)) + { + return true; + } + + var parameter = command.Parameters.Cast().FirstOrDefault( + p => string.Equals(p.ParameterName, token, StringComparison.Ordinal) + || string.Equals(p.ParameterName.TrimStart('@'), token.TrimStart('@'), StringComparison.Ordinal)); + if (parameter?.Value is null || parameter.Value == DBNull.Value) + { + return false; + } + + try + { + value = Convert.ToInt64(parameter.Value, CultureInfo.InvariantCulture); + return true; + } + catch (Exception ex) when (ex is FormatException or InvalidCastException or OverflowException) + { + return false; + } + } + + private static async Task MaterializeTypedAsync(DbDataReader reader, CancellationToken ct) + { + var table = CreateTypedTable(reader); + var values = new object[reader.FieldCount]; + while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + reader.GetValues(values); + table.Rows.Add((object[])values.Clone()); + } + + return table; + } + + private static DataTable MaterializeTyped(DbDataReader reader) + { + var table = CreateTypedTable(reader); + var values = new object[reader.FieldCount]; + while (reader.Read()) + { + reader.GetValues(values); + table.Rows.Add((object[])values.Clone()); + } + + return table; + } + + private static DataTable CreateTypedTable(DbDataReader reader) + { + var table = new DataTable(); + var names = new HashSet(StringComparer.Ordinal); + for (var ordinal = 0; ordinal < reader.FieldCount; ordinal++) + { + var name = reader.GetName(ordinal); + if (string.IsNullOrWhiteSpace(name) || !names.Add(name)) + { + name = string.Concat("Column", ordinal.ToString(CultureInfo.InvariantCulture)); + names.Add(name); + } + + table.Columns.Add(name, reader.GetFieldType(ordinal)); + } + + return table; + } + // ───────────────────────────────────────────────────────────────────────── // ORDER BY RANDOM() → TABLESAMPLE rewrite // ───────────────────────────────────────────────────────────────────────── @@ -534,7 +752,7 @@ private async ValueTask> RewriteRandomAndExecut // rewritten is produced entirely by RewriteRandomSql from EF Core SQL, // never from user input. using var newCmd = command.Connection!.CreateCommand(); - SetRewrittenCommandText(newCmd, rewritten); + SetGeneratedCommandText(newCmd, rewritten); newCmd.Transaction = command.Transaction; foreach (DbParameter p in command.Parameters) { @@ -550,23 +768,22 @@ private async ValueTask> RewriteRandomAndExecut await newCmd.Connection.OpenAsync(ct).ConfigureAwait(false); } - var dt = new DataTable(); var readerObj = await newCmd.ExecuteReaderAsync(ct).ConfigureAwait(false); await using (readerObj.ConfigureAwait(false)) { - dt.Load(readerObj); - } + var dt = await MaterializeTypedAsync(readerObj, ct).ConfigureAwait(false); - if (_logger?.IsEnabled(LogLevel.Debug) == true) - { - _logger?.LogDebug("HomeQueryCache RANDOM rewritten with TABLESAMPLE: {Rows} rows", dt.Rows.Count); - } + if (_logger?.IsEnabled(LogLevel.Debug) == true) + { + _logger?.LogDebug("HomeQueryCache RANDOM rewritten with TABLESAMPLE: {Rows} rows", dt.Rows.Count); + } - return InterceptionResult.SuppressWithResult(new DataTableReader(dt)); + return InterceptionResult.SuppressWithResult(new DataTableReader(dt)); + } } catch (Exception ex) { - _logger?.LogWarning(ex, "ORDER BY RANDOM() rewrite failed — falling back to original query."); + PostgresLog.Warn("[Cache] La optimización ORDER BY RANDOM() falló; se usará la consulta original.", ex); return default; } } @@ -611,8 +828,145 @@ private async ValueTask> RewriteRandomAndExecut return string.Concat(rewritten.AsSpan(0, idx), replacement, rewritten.AsSpan(idx + table.Length)); } - // CA2100: sql comes from RewriteRandomSql which only transforms EF Core-generated SQL, never from user input. + // CA2100: sql comes from either RewriteRandomSql or a captured EF Core command, never from raw user input. [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "sql is produced by RewriteRandomSql from EF Core-generated SQL, never from raw user HTTP input.")] - private static void SetRewrittenCommandText(System.Data.Common.DbCommand cmd, string sql) + private static void SetGeneratedCommandText(System.Data.Common.DbCommand cmd, string sql) => cmd.CommandText = sql; + + private readonly record struct PagePrefetchPlan(string OffsetParameterName, long NextOffset, int Limit); + + private sealed class CachedResult + { + public CachedResult(DataTable table, bool isPrefetched) + { + Table = table; + IsPrefetched = isPrefetched; + } + + public DataTable Table { get; } + + public bool IsPrefetched { get; } + } + + private sealed class CommandSnapshot + { + private CommandSnapshot(string connectionString, string commandText, ParameterSnapshot[] parameters, string cacheKey, long offset) + { + ConnectionString = connectionString; + CommandText = commandText; + Parameters = parameters; + CacheKey = cacheKey; + Offset = offset; + } + + public string ConnectionString { get; } + + public string CommandText { get; } + + public ParameterSnapshot[] Parameters { get; } + + public string CacheKey { get; } + + public long Offset { get; } + + public static CommandSnapshot? Create(DbCommand source, PagePrefetchPlan plan) + { + var connectionString = source.Connection?.ConnectionString; + if (string.IsNullOrWhiteSpace(connectionString)) + { + return null; + } + + var parameters = source.Parameters.Cast() + .Select(parameter => ParameterSnapshot.Create(parameter, plan)) + .ToArray(); + + using var connection = new NpgsqlConnection(connectionString); + using var command = connection.CreateCommand(); + SetGeneratedCommandText(command, source.CommandText); + foreach (var parameter in parameters) + { + command.Parameters.Add(parameter.Create(command)); + } + + var cacheKey = BuildCacheKey(command); + if (cacheKey is null) + { + return null; + } + + return new CommandSnapshot(connectionString, source.CommandText, parameters, cacheKey, plan.NextOffset); + } + } + + private sealed class ParameterSnapshot + { + private ParameterSnapshot(DbParameter source, object? value) + { + Name = source.ParameterName; + Value = value; + DbType = source.DbType; + Direction = source.Direction; + Size = source.Size; + Precision = source.Precision; + Scale = source.Scale; + IsNullable = source.IsNullable; + NpgsqlDbType = source is NpgsqlParameter npgsql ? npgsql.NpgsqlDbType : null; + DataTypeName = source is NpgsqlParameter typed ? typed.DataTypeName : null; + } + + public string Name { get; } + + public object? Value { get; } + + public System.Data.DbType DbType { get; } + + public ParameterDirection Direction { get; } + + public int Size { get; } + + public byte Precision { get; } + + public byte Scale { get; } + + public bool IsNullable { get; } + + public NpgsqlTypes.NpgsqlDbType? NpgsqlDbType { get; } + + public string? DataTypeName { get; } + + public static ParameterSnapshot Create(DbParameter source, PagePrefetchPlan plan) + { + var isOffset = string.Equals(source.ParameterName, plan.OffsetParameterName, StringComparison.Ordinal) + || string.Equals(source.ParameterName.TrimStart('@'), plan.OffsetParameterName.TrimStart('@'), StringComparison.Ordinal); + return new ParameterSnapshot(source, isOffset ? plan.NextOffset : source.Value); + } + + public DbParameter Create(DbCommand command) + { + var parameter = command.CreateParameter(); + parameter.ParameterName = Name; + parameter.Value = Value ?? DBNull.Value; + parameter.DbType = DbType; + parameter.Direction = Direction; + parameter.Size = Size; + parameter.Precision = Precision; + parameter.Scale = Scale; + parameter.IsNullable = IsNullable; + if (parameter is NpgsqlParameter npgsql) + { + if (NpgsqlDbType.HasValue) + { + npgsql.NpgsqlDbType = NpgsqlDbType.Value; + } + + if (DataTypeName is not null) + { + npgsql.DataTypeName = DataTypeName; + } + } + + return parameter; + } + } } diff --git a/Jellyfin.Database.Providers.Postgres/Services/JellyTrendPostgresStore.cs b/Jellyfin.Database.Providers.Postgres/Services/JellyTrendPostgresStore.cs deleted file mode 100644 index 98bb3cb..0000000 --- a/Jellyfin.Database.Providers.Postgres/Services/JellyTrendPostgresStore.cs +++ /dev/null @@ -1,835 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -using Jellyfin.Database.Providers.Postgres.Logging; -using Jellyfin.Plugin.JellyTrend.Api; -using MediaBrowser.Common.Configuration; -using Microsoft.Extensions.Logging; -using Npgsql; -using NpgsqlTypes; - -namespace Jellyfin.Database.Providers.Postgres.Services; - -/// -/// Persists JellyTrend's own data (feature cache, trending list, recommendations, history) in a -/// dedicated jellytrend schema of the Jellyfin database. -/// -/// -/// -/// The schema lives outside public on purpose: the PostgreSQL to SQLite export enumerates -/// pg_tables of the public schema, so these tables can never end up inside a SQLite file -/// that Jellyfin would open without them. -/// -/// -/// Nothing is created until JellyTrend calls , so a server without that plugin -/// never gets these tables. The features of an item are stored as one jsonb document instead of -/// modelling every facet in SQL: the provider does not need to understand them to keep them, and a GIN -/// index still allows querying inside the document later. -/// -/// -/// No method throws: a broken store degrades to "no data" so JellyTrend can fall back to its JSON files -/// and the task still finishes. -/// -/// -public sealed class JellyTrendPostgresStore : IJellyTrendStoreProvider -{ - /// Version of the schema this class creates. - public const int CurrentSchemaVersion = 4; - - private const string SchemaName = "jellytrend"; - - private const string SchemaSql = """ - CREATE SCHEMA IF NOT EXISTS jellytrend; - - CREATE TABLE IF NOT EXISTS jellytrend.schema_version ( - version integer PRIMARY KEY, - applied_at timestamptz NOT NULL DEFAULT now()); - - CREATE TABLE IF NOT EXISTS jellytrend.item_features ( - item_id uuid PRIMARY KEY, - facets jsonb NOT NULL, - content_hash text NOT NULL, - computed_at timestamptz NOT NULL DEFAULT now()); - - CREATE INDEX IF NOT EXISTS ix_jt_item_features_facets - ON jellytrend.item_features USING gin (facets); - - CREATE TABLE IF NOT EXISTS jellytrend.trending_item ( - item_id uuid PRIMARY KEY, - entry jsonb NOT NULL, - media_type text, - synced_at timestamptz NOT NULL); - - CREATE TABLE IF NOT EXISTS jellytrend.user_recommendation ( - user_id uuid NOT NULL, - item_id uuid NOT NULL, - rank integer NOT NULL, - score double precision NOT NULL DEFAULT 0, - run_id uuid NOT NULL, - generated_at timestamptz NOT NULL, - PRIMARY KEY (user_id, item_id)); - - CREATE INDEX IF NOT EXISTS ix_jt_user_recommendation_rank - ON jellytrend.user_recommendation (user_id, rank); - - CREATE TABLE IF NOT EXISTS jellytrend.user_suppression ( - user_id uuid NOT NULL, - item_id uuid NOT NULL, - reason text NOT NULL, - PRIMARY KEY (user_id, item_id)); - - CREATE TABLE IF NOT EXISTS jellytrend.sync_run ( - run_id uuid PRIMARY KEY, - kind text NOT NULL, - started_at timestamptz NOT NULL, - finished_at timestamptz, - status text, - users_ok integer NOT NULL DEFAULT 0, - users_failed integer NOT NULL DEFAULT 0, - duration_ms integer, - message text); - - CREATE TABLE IF NOT EXISTS jellytrend.user_item_consumption ( - user_id uuid NOT NULL, - item_id uuid NOT NULL, - data jsonb NOT NULL, - updated_at timestamptz NOT NULL DEFAULT now(), - PRIMARY KEY (user_id, item_id)); - - CREATE TABLE IF NOT EXISTS jellytrend.user_affinity ( - user_id uuid NOT NULL, - facet text NOT NULL, - value text NOT NULL, - weight real NOT NULL, - updated_at timestamptz NOT NULL DEFAULT now(), - PRIMARY KEY (user_id, facet, value)); - - CREATE TABLE IF NOT EXISTS jellytrend.user_affinity_pair ( - user_id uuid NOT NULL, - facet text NOT NULL, - value text NOT NULL, - paired_facet text NOT NULL, - paired_value text NOT NULL, - weight real NOT NULL, - updated_at timestamptz NOT NULL DEFAULT now(), - PRIMARY KEY (user_id, facet, value, paired_facet, paired_value)); - """; - - private readonly IApplicationPaths _applicationPaths; - - /// - /// Initializes a new instance of the class. - /// - /// Application paths, used to read the active connection string. - /// Logger factory supplied by the server. - public JellyTrendPostgresStore(IApplicationPaths applicationPaths, ILoggerFactory loggerFactory) - { - _applicationPaths = applicationPaths; - _ = loggerFactory; - } - - /// - /// Gets the connection string Jellyfin is using right now, or null when PostgreSQL is not active. - /// - private string? ConnectionString - => PostgresPlugin.ReadActivePgConnectionString(_applicationPaths); - - /// - public bool EnsureSchema() - { - PostgresLog.Info($"[JellyTrend] Preparando el esquema '{SchemaName}' para los datos del plugin."); - - var created = Run( - conn => - { - using var command = new NpgsqlCommand(SchemaSql, conn) { CommandTimeout = 120 }; - command.ExecuteNonQuery(); - - using var version = new NpgsqlCommand( - "INSERT INTO jellytrend.schema_version (version) VALUES (@v) ON CONFLICT (version) DO NOTHING;", - conn); - version.Parameters.AddWithValue("v", CurrentSchemaVersion); - version.ExecuteNonQuery(); - return true; - }, - false, - "EnsureSchema"); - - PostgresLog.Info(created - ? $"[JellyTrend] Esquema listo ({DescribeBackend()})." - : "[JellyTrend] No se pudo preparar el esquema; se usaran los archivos JSON."); - - return created; - } - - /// - public int GetSchemaVersion() - => Run( - conn => - { - using var command = new NpgsqlCommand("SELECT max(version) FROM jellytrend.schema_version;", conn); - var value = command.ExecuteScalar(); - return value is null or DBNull ? 0 : Convert.ToInt32(value, System.Globalization.CultureInfo.InvariantCulture); - }, - 0, - "GetSchemaVersion"); - - /// - public string DescribeBackend() - => Run( - conn => - { - using var command = new NpgsqlCommand( - $"SELECT 'PostgreSQL ' || current_setting('server_version') || ', esquema {SchemaName}';", - conn); - return command.ExecuteScalar() as string ?? $"esquema {SchemaName}"; - }, - $"esquema {SchemaName} (sin conexion)", - "DescribeBackend"); - - /// - public int ReplaceItemFeatures(Guid[] itemIds, string[] facetsJson, string[] contentHashes) - { - ArgumentNullException.ThrowIfNull(itemIds); - - if (itemIds.Length == 0) - { - return 0; - } - - return Run( - conn => - { - using var transaction = conn.BeginTransaction(); - var written = 0; - - using var command = new NpgsqlCommand( - """ - INSERT INTO jellytrend.item_features (item_id, facets, content_hash, computed_at) - VALUES (@id, @facets, @hash, now()) - ON CONFLICT (item_id) DO UPDATE - SET facets = EXCLUDED.facets, - content_hash = EXCLUDED.content_hash, - computed_at = now(); - """, - conn, - transaction); - - var idParameter = command.Parameters.Add("id", NpgsqlDbType.Uuid); - var facetsParameter = command.Parameters.Add("facets", NpgsqlDbType.Jsonb); - var hashParameter = command.Parameters.Add("hash", NpgsqlDbType.Text); - - for (var i = 0; i < itemIds.Length; i++) - { - if (itemIds[i] == Guid.Empty) - { - continue; - } - - idParameter.Value = itemIds[i]; - facetsParameter.Value = i < facetsJson.Length ? facetsJson[i] : "{}"; - hashParameter.Value = i < contentHashes.Length ? contentHashes[i] : string.Empty; - written += command.ExecuteNonQuery(); - } - - transaction.Commit(); - return written; - }, - 0, - "ReplaceItemFeatures"); - } - - /// - public string?[] GetItemFeatures(Guid[] itemIds) - { - ArgumentNullException.ThrowIfNull(itemIds); - - var missing = new string?[itemIds.Length]; - - if (itemIds.Length == 0) - { - return missing; - } - - var found = Run( - conn => - { - using var command = new NpgsqlCommand( - "SELECT item_id, facets FROM jellytrend.item_features WHERE item_id = ANY(@ids);", - conn); - command.Parameters.AddWithValue("ids", itemIds); - - var rows = new Dictionary(); - using var reader = command.ExecuteReader(); - while (reader.Read()) - { - rows[reader.GetGuid(0)] = reader.GetString(1); - } - - return rows; - }, - new Dictionary(), - "GetItemFeatures"); - - for (var i = 0; i < itemIds.Length; i++) - { - if (found.TryGetValue(itemIds[i], out var facets)) - { - missing[i] = facets; - } - } - - return missing; - } - - /// - public string? GetItemFeaturesJson() - => Run( - conn => - { - // El documento lo arma PostgreSQL: una sola consulta y sin serializar a mano. - using var command = new NpgsqlCommand( - "SELECT jsonb_object_agg(item_id::text, facets) FROM jellytrend.item_features;", - conn); - var value = command.ExecuteScalar(); - return value is null or DBNull ? null : value as string; - }, - null, - "GetItemFeaturesJson"); - - /// - public void ReplaceTrending(Guid[] itemIds, string[] entriesJson, DateTime syncedAt) - { - ArgumentNullException.ThrowIfNull(itemIds); - - Run( - conn => - { - using var transaction = conn.BeginTransaction(); - - using (var clear = new NpgsqlCommand("DELETE FROM jellytrend.trending_item;", conn, transaction)) - { - clear.ExecuteNonQuery(); - } - - using (var command = new NpgsqlCommand( - """ - INSERT INTO jellytrend.trending_item (item_id, entry, media_type, synced_at) - VALUES (@id, @entry, @media, @synced) - ON CONFLICT (item_id) DO UPDATE - SET entry = EXCLUDED.entry, - media_type = EXCLUDED.media_type, - synced_at = EXCLUDED.synced_at; - """, - conn, - transaction)) - { - var idParameter = command.Parameters.Add("id", NpgsqlDbType.Uuid); - var entryParameter = command.Parameters.Add("entry", NpgsqlDbType.Jsonb); - var mediaParameter = command.Parameters.Add("media", NpgsqlDbType.Text); - var syncedParameter = command.Parameters.Add("synced", NpgsqlDbType.TimestampTz); - syncedParameter.Value = DateTime.SpecifyKind(syncedAt, DateTimeKind.Utc); - - for (var i = 0; i < itemIds.Length; i++) - { - if (itemIds[i] == Guid.Empty) - { - continue; - } - - idParameter.Value = itemIds[i]; - entryParameter.Value = i < entriesJson.Length ? entriesJson[i] : "{}"; - mediaParameter.Value = ReadMediaType(entriesJson, i); - command.ExecuteNonQuery(); - } - } - - transaction.Commit(); - return true; - }, - false, - "ReplaceTrending"); - } - - /// - public string? GetTrendingJson() - => Run( - conn => - { - // PostgreSQL arma el documento: mismas claves que el archivo JSON, sin serializar a mano. - using var command = new NpgsqlCommand( - """ - SELECT jsonb_build_object( - 'Items', jsonb_agg(entry ORDER BY item_id), - 'LastUpdated', max(synced_at)) - FROM jellytrend.trending_item; - """, - conn); - var value = command.ExecuteScalar(); - return value is null or DBNull ? null : value as string; - }, - null, - "GetTrendingJson"); - - /// - public void ReplaceUserRecommendations(Guid userId, Guid[] itemIds, double[] scores, DateTime generatedAt, Guid runId) - { - ArgumentNullException.ThrowIfNull(itemIds); - - Run( - conn => - { - using var transaction = conn.BeginTransaction(); - - using (var clear = new NpgsqlCommand( - "DELETE FROM jellytrend.user_recommendation WHERE user_id = @user;", - conn, - transaction)) - { - clear.Parameters.AddWithValue("user", userId); - clear.ExecuteNonQuery(); - } - - using (var command = new NpgsqlCommand( - """ - INSERT INTO jellytrend.user_recommendation (user_id, item_id, rank, score, run_id, generated_at) - VALUES (@user, @item, @rank, @score, @run, @generated) - ON CONFLICT (user_id, item_id) DO UPDATE - SET rank = EXCLUDED.rank, - score = EXCLUDED.score, - run_id = EXCLUDED.run_id, - generated_at = EXCLUDED.generated_at; - """, - conn, - transaction)) - { - var itemParameter = command.Parameters.Add("item", NpgsqlDbType.Uuid); - var rankParameter = command.Parameters.Add("rank", NpgsqlDbType.Integer); - var scoreParameter = command.Parameters.Add("score", NpgsqlDbType.Double); - command.Parameters.AddWithValue("user", userId); - command.Parameters.AddWithValue("run", runId); - command.Parameters.AddWithValue("generated", DateTime.SpecifyKind(generatedAt, DateTimeKind.Utc)); - - for (var i = 0; i < itemIds.Length; i++) - { - if (itemIds[i] == Guid.Empty) - { - continue; - } - - itemParameter.Value = itemIds[i]; - rankParameter.Value = i; - scoreParameter.Value = i < scores.Length ? scores[i] : 0d; - command.ExecuteNonQuery(); - } - } - - transaction.Commit(); - return true; - }, - false, - "ReplaceUserRecommendations"); - } - - /// - public Guid[] GetUserRecommendations(Guid userId) - => Run( - conn => - { - using var command = new NpgsqlCommand( - "SELECT item_id FROM jellytrend.user_recommendation WHERE user_id = @user ORDER BY rank;", - conn); - command.Parameters.AddWithValue("user", userId); - return ReadGuids(command); - }, - [], - "GetUserRecommendations"); - - /// - public Guid[] GetUsersWithRecommendations() - => Run( - conn => - { - using var command = new NpgsqlCommand( - "SELECT user_id FROM jellytrend.user_recommendation GROUP BY user_id ORDER BY max(generated_at) DESC;", - conn); - return ReadGuids(command); - }, - [], - "GetUsersWithRecommendations"); - - /// - public void ReplaceSuppressed(Guid userId, Guid[] itemIds, string[] reasons) - { - ArgumentNullException.ThrowIfNull(itemIds); - - Run( - conn => - { - using var transaction = conn.BeginTransaction(); - - using (var clear = new NpgsqlCommand( - "DELETE FROM jellytrend.user_suppression WHERE user_id = @user;", - conn, - transaction)) - { - clear.Parameters.AddWithValue("user", userId); - clear.ExecuteNonQuery(); - } - - using (var command = new NpgsqlCommand( - """ - INSERT INTO jellytrend.user_suppression (user_id, item_id, reason) - VALUES (@user, @item, @reason) - ON CONFLICT (user_id, item_id) DO UPDATE SET reason = EXCLUDED.reason; - """, - conn, - transaction)) - { - var itemParameter = command.Parameters.Add("item", NpgsqlDbType.Uuid); - var reasonParameter = command.Parameters.Add("reason", NpgsqlDbType.Text); - command.Parameters.AddWithValue("user", userId); - - for (var i = 0; i < itemIds.Length; i++) - { - if (itemIds[i] == Guid.Empty) - { - continue; - } - - itemParameter.Value = itemIds[i]; - reasonParameter.Value = i < reasons.Length ? reasons[i] : "played"; - command.ExecuteNonQuery(); - } - } - - transaction.Commit(); - return true; - }, - false, - "ReplaceSuppressed"); - } - - /// - public Guid[] GetSuppressed(Guid userId) - => Run( - conn => - { - using var command = new NpgsqlCommand( - "SELECT item_id FROM jellytrend.user_suppression WHERE user_id = @user;", - conn); - command.Parameters.AddWithValue("user", userId); - return ReadGuids(command); - }, - [], - "GetSuppressed"); - - /// - public int ReplaceConsumption(Guid userId, Guid[] itemIds, string[] consumptionJson) - { - ArgumentNullException.ThrowIfNull(itemIds); - - if (itemIds.Length == 0) - { - return 0; - } - - return Run( - conn => - { - using var transaction = conn.BeginTransaction(); - var written = 0; - - using var command = new NpgsqlCommand( - """ - INSERT INTO jellytrend.user_item_consumption (user_id, item_id, data, updated_at) - VALUES (@user, @item, @data, now()) - ON CONFLICT (user_id, item_id) DO UPDATE - SET data = EXCLUDED.data, - updated_at = now(); - """, - conn, - transaction); - - command.Parameters.AddWithValue("user", userId); - var itemParameter = command.Parameters.Add("item", NpgsqlDbType.Uuid); - var dataParameter = command.Parameters.Add("data", NpgsqlDbType.Jsonb); - - for (var i = 0; i < itemIds.Length; i++) - { - if (itemIds[i] == Guid.Empty) - { - continue; - } - - itemParameter.Value = itemIds[i]; - dataParameter.Value = i < consumptionJson.Length ? consumptionJson[i] : "{}"; - written += command.ExecuteNonQuery(); - } - - transaction.Commit(); - return written; - }, - 0, - "ReplaceConsumption"); - } - - /// - public string? GetUserConsumption(Guid userId) - => Run( - conn => - { - using var command = new NpgsqlCommand( - "SELECT jsonb_object_agg(item_id::text, data) FROM jellytrend.user_item_consumption WHERE user_id = @user;", - conn); - command.Parameters.AddWithValue("user", userId); - var value = command.ExecuteScalar(); - return value is null or DBNull ? null : value as string; - }, - null, - "GetUserConsumption"); - - /// - public int ReplaceAffinities(Guid userId, string[] facets, string[] values, string[] pairedFacets, string[] pairedValues, double[] weights) - { - ArgumentNullException.ThrowIfNull(facets); - - if (facets.Length == 0) - { - return 0; - } - - return Run( - conn => - { - using var transaction = conn.BeginTransaction(); - - using (var clear = new NpgsqlCommand( - "DELETE FROM jellytrend.user_affinity WHERE user_id = @user; DELETE FROM jellytrend.user_affinity_pair WHERE user_id = @user;", - conn, - transaction)) - { - clear.Parameters.AddWithValue("user", userId); - clear.ExecuteNonQuery(); - } - - var written = 0; - - using (var single = new NpgsqlCommand( - """ - INSERT INTO jellytrend.user_affinity (user_id, facet, value, weight, updated_at) - VALUES (@user, @facet, @value, @weight, now()) - ON CONFLICT (user_id, facet, value) DO UPDATE SET weight = EXCLUDED.weight, updated_at = now(); - """, - conn, - transaction)) - using (var pair = new NpgsqlCommand( - """ - INSERT INTO jellytrend.user_affinity_pair (user_id, facet, value, paired_facet, paired_value, weight, updated_at) - VALUES (@user, @facet, @value, @pairedFacet, @pairedValue, @weight, now()) - ON CONFLICT (user_id, facet, value, paired_facet, paired_value) DO UPDATE SET weight = EXCLUDED.weight, updated_at = now(); - """, - conn, - transaction)) - { - var singleFacet = single.Parameters.Add("facet", NpgsqlDbType.Text); - var singleValue = single.Parameters.Add("value", NpgsqlDbType.Text); - var singleWeight = single.Parameters.Add("weight", NpgsqlDbType.Real); - single.Parameters.AddWithValue("user", userId); - - var pairFacet = pair.Parameters.Add("facet", NpgsqlDbType.Text); - var pairValue = pair.Parameters.Add("value", NpgsqlDbType.Text); - var pairPairedFacet = pair.Parameters.Add("pairedFacet", NpgsqlDbType.Text); - var pairPairedValue = pair.Parameters.Add("pairedValue", NpgsqlDbType.Text); - var pairWeight = pair.Parameters.Add("weight", NpgsqlDbType.Real); - pair.Parameters.AddWithValue("user", userId); - - var weight = 0d; - - for (var i = 0; i < facets.Length; i++) - { - if (string.IsNullOrWhiteSpace(facets[i]) || string.IsNullOrWhiteSpace(values[i])) - { - continue; - } - - weight = i < weights.Length ? weights[i] : 0d; - var paired = i < pairedFacets.Length && !string.IsNullOrWhiteSpace(pairedFacets[i]); - - if (paired) - { - pairFacet.Value = facets[i]; - pairValue.Value = values[i]; - pairPairedFacet.Value = pairedFacets[i]; - pairPairedValue.Value = i < pairedValues.Length ? pairedValues[i] : string.Empty; - pairWeight.Value = (float)weight; - written += pair.ExecuteNonQuery(); - } - else - { - singleFacet.Value = facets[i]; - singleValue.Value = values[i]; - singleWeight.Value = (float)weight; - written += single.ExecuteNonQuery(); - } - } - } - - transaction.Commit(); - return written; - }, - 0, - "ReplaceAffinities"); - } - - /// - public string? GetAffinities(Guid userId) - => Run( - conn => - { - using var command = new NpgsqlCommand( - """ - SELECT coalesce(jsonb_agg(entry), '[]'::jsonb) - FROM ( - SELECT jsonb_build_object('Facet', facet, 'Value', value, 'Weight', weight) AS entry - FROM jellytrend.user_affinity WHERE user_id = @user - UNION ALL - SELECT jsonb_build_object('Facet', facet, 'Value', value, - 'PairedFacet', paired_facet, 'PairedValue', paired_value, - 'Weight', weight) - FROM jellytrend.user_affinity_pair WHERE user_id = @user - ) parts; - """, - conn); - command.Parameters.AddWithValue("user", userId); - var value = command.ExecuteScalar(); - return value is null or DBNull ? null : value as string; - }, - null, - "GetAffinities"); - - /// - public Guid StartRun(string kind, DateTime startedAt) - { - var runId = Guid.NewGuid(); - - Run( - conn => - { - using var command = new NpgsqlCommand( - """ - INSERT INTO jellytrend.sync_run (run_id, kind, started_at, status) - VALUES (@run, @kind, @started, 'running'); - """, - conn); - command.Parameters.AddWithValue("run", runId); - command.Parameters.AddWithValue("kind", kind); - command.Parameters.AddWithValue("started", DateTime.SpecifyKind(startedAt, DateTimeKind.Utc)); - command.ExecuteNonQuery(); - return true; - }, - false, - "StartRun"); - - return runId; - } - - /// - public void FinishRun(Guid runId, string status, int usersOk, int usersFailed, int durationMs, string? message) - => Run( - conn => - { - using var command = new NpgsqlCommand( - """ - UPDATE jellytrend.sync_run - SET finished_at = now(), - status = @status, - users_ok = @ok, - users_failed = @failed, - duration_ms = @duration, - message = @message - WHERE run_id = @run; - """, - conn); - command.Parameters.AddWithValue("run", runId); - command.Parameters.AddWithValue("status", status); - command.Parameters.AddWithValue("ok", usersOk); - command.Parameters.AddWithValue("failed", usersFailed); - command.Parameters.AddWithValue("duration", durationMs); - command.Parameters.AddWithValue("message", (object?)message ?? DBNull.Value); - command.ExecuteNonQuery(); - return true; - }, - false, - "FinishRun"); - - private static Guid[] ReadGuids(NpgsqlCommand command) - { - var ids = new List(); - using var reader = command.ExecuteReader(); - while (reader.Read()) - { - ids.Add(reader.GetGuid(0)); - } - - return [.. ids]; - } - - private static string? ReadMediaType(string[] entriesJson, int index) - { - if (index >= entriesJson.Length) - { - return null; - } - - // El entry ya trae su media type: se guarda aparte solo para poder filtrar por tipo en SQL. - var marker = "\"MediaType\":"; - var at = entriesJson[index].IndexOf(marker, StringComparison.OrdinalIgnoreCase); - if (at < 0) - { - return null; - } - - var rest = entriesJson[index][(at + marker.Length)..].TrimStart(); - if (rest.StartsWith('"')) - { - var end = rest.IndexOf('"', 1); - return end > 1 ? rest[1..end] : null; - } - - var digits = new string([.. rest.TakeWhile(char.IsDigit)]); - return digits.Length == 0 ? null : digits; - } - - private T Run(Func action, T fallback, string operation) - { - var connectionString = ConnectionString; - if (string.IsNullOrWhiteSpace(connectionString)) - { - return fallback; - } - - try - { - using var connection = new NpgsqlConnection(connectionString); - connection.Open(); - return action(connection); - } - catch (Exception ex) - { - PostgresLog.Warn($"[JellyTrend] {operation} fallo; se sigue con el almacen JSON: {ex.Message}"); - return fallback; - } - } - - private bool Run(Action action, string operation) - => Run( - connection => - { - action(connection); - return null; - }, - null, - operation) is null; -} diff --git a/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs b/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs new file mode 100644 index 0000000..ef0a272 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs @@ -0,0 +1,410 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Database.Providers.Postgres.Api; +using Jellyfin.Database.Providers.Postgres.Logging; +using MediaBrowser.Common.Configuration; +using Npgsql; + +namespace Jellyfin.Database.Providers.Postgres.Services; + +/// +/// Implements the private-schema storage contract without leaking PostgreSQL credentials or connections. +/// +[SuppressMessage( + "Security", + "CA2100:Review SQL queries for security vulnerabilities", + Justification = "Schema identifiers are generated exclusively from a GUID. Consumer SQL is validated as one unqualified statement before command construction, and all values are bound parameters.")] +internal sealed partial class PostgresPluginSchemaHost : IPluginSchemaHost +{ + private const string MigrationTable = "__pg_provider_schema_migrations"; + private const string RegistrySchema = "jellyfin_provider"; + private const int MaxSqlLength = 65_536; + + private readonly Func _connectionStringFactory; + + /// + /// Initializes a new instance of the class using Jellyfin's active provider configuration. + /// + /// Application paths used to read the active database configuration. + public PostgresPluginSchemaHost(IApplicationPaths applicationPaths) + : this(() => PostgresPlugin.ReadActivePgConnectionString(applicationPaths) + ?? PostgresPlugin.Instance?.Configuration.ConnectionString) + { + } + + internal PostgresPluginSchemaHost(Func connectionStringFactory) + { + _connectionStringFactory = connectionStringFactory ?? throw new ArgumentNullException(nameof(connectionStringFactory)); + } + + /// + public async Task OpenSchemaAsync(Guid pluginId, CancellationToken cancellationToken = default) + => await OpenSchemaAsync( + new PluginSchemaRequest(pluginId, string.Concat("plugin_", pluginId.ToString("N", CultureInfo.InvariantCulture))), + cancellationToken).ConfigureAwait(false); + + /// + public async Task OpenSchemaAsync(PluginSchemaRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + if (request.PluginId == Guid.Empty) + { + throw new ArgumentException("A plugin identifier is required.", nameof(request)); + } + + ValidateSchemaName(request.SchemaName); + using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + try + { + await EnsureRegistryAsync(connection, transaction, cancellationToken).ConfigureAwait(false); + var existed = await SchemaExistsAsync(connection, transaction, request.SchemaName, cancellationToken).ConfigureAwait(false); + using (var command = new NpgsqlCommand($"CREATE SCHEMA IF NOT EXISTS {QuoteIdentifier(request.SchemaName)};", connection, transaction)) + { + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + var schema = new PluginSchema(request.PluginId, request.SchemaName); + await RegisterSchemaAsync(connection, transaction, request, existed, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + PostgresLog.Info($"[PluginSchema] Esquema privado disponible: {request.SchemaName}."); + return schema; + } + catch + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + /// + public async Task ApplyMigrationsAsync( + PluginSchema schema, + IReadOnlyList migrations, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(schema); + ArgumentNullException.ThrowIfNull(migrations); + ValidateSchema(schema); + ValidateMigrations(migrations); + + using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + try + { + await SetSchemaAsync(connection, transaction, schema, cancellationToken).ConfigureAwait(false); + await EnsureMigrationTableAsync(connection, transaction, cancellationToken).ConfigureAwait(false); + await AcquireMigrationLockAsync(connection, transaction, schema, cancellationToken).ConfigureAwait(false); + + foreach (var migration in migrations) + { + var appliedChecksum = await GetAppliedChecksumAsync(connection, transaction, migration.Id, cancellationToken).ConfigureAwait(false); + if (appliedChecksum is not null) + { + if (!string.Equals(appliedChecksum, migration.Checksum, StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Applied migration '{migration.Id}' was modified."); + } + + continue; + } + + await ExecuteInTransactionAsync(connection, transaction, migration.Sql, null, cancellationToken).ConfigureAwait(false); + await RecordMigrationAsync(connection, transaction, migration, cancellationToken).ConfigureAwait(false); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + PostgresLog.Info($"[PluginSchema] Migraciones aplicadas para {schema.Name}: {migrations.Count} declaradas."); + } + catch (Exception ex) + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + PostgresLog.Error($"[PluginSchema] Migración fallida para {schema.Name}", ex); + throw; + } + } + + /// + public async Task ExecuteAsync( + PluginSchema schema, + string sql, + IReadOnlyList? parameters = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(schema); + ValidateSchema(schema); + ValidateDataSql(sql); + + using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + try + { + await SetSchemaAsync(connection, transaction, schema, cancellationToken).ConfigureAwait(false); + var affected = await ExecuteInTransactionAsync(connection, transaction, sql, parameters, cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return affected; + } + catch (Exception ex) + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + PostgresLog.Error($"[PluginSchema] Escritura fallida para {schema.Name}", ex); + throw; + } + } + + /// + public async Task QueryAsync( + PluginSchema schema, + string sql, + Func project, + IReadOnlyList? parameters = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(schema); + ArgumentNullException.ThrowIfNull(project); + ValidateSchema(schema); + ValidateQuerySql(sql); + + using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var transaction = await connection.BeginTransactionAsync(IsolationLevel.ReadCommitted, cancellationToken).ConfigureAwait(false); + try + { + await SetSchemaAsync(connection, transaction, schema, cancellationToken).ConfigureAwait(false); + T result; + using (var command = CreateCommand(connection, transaction, sql, parameters)) + using (var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)) + { + result = project(reader); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return result; + } + catch (Exception ex) + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + PostgresLog.Error($"[PluginSchema] Consulta fallida para {schema.Name}", ex); + throw; + } + } + + private static async Task SchemaExistsAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, string schemaName, CancellationToken cancellationToken) + { + using var command = new NpgsqlCommand("SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = @schema);", connection, transaction); + command.Parameters.AddWithValue("schema", schemaName); + return (bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!; + } + + private static async Task EnsureRegistryAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, CancellationToken cancellationToken) + { + using (var schema = new NpgsqlCommand($"CREATE SCHEMA IF NOT EXISTS {QuoteIdentifier(RegistrySchema)};", connection, transaction)) + { + await schema.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + const string sql = "CREATE TABLE IF NOT EXISTS jellyfin_provider.plugin_schemas (plugin_id uuid PRIMARY KEY, plugin_name text NOT NULL, schema_name text NOT NULL UNIQUE, protocol_version integer NOT NULL, created_at timestamp with time zone NOT NULL DEFAULT now(), last_seen_at timestamp with time zone NOT NULL DEFAULT now(), state text NOT NULL DEFAULT 'active')"; + await ExecuteInTransactionAsync(connection, transaction, sql, null, cancellationToken).ConfigureAwait(false); + } + + private static async Task RegisterSchemaAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, PluginSchemaRequest request, bool existed, CancellationToken cancellationToken) + { + const string lookupSql = "SELECT plugin_id, schema_name FROM jellyfin_provider.plugin_schemas WHERE plugin_id = @pluginId OR schema_name = @schema;"; + using var lookup = new NpgsqlCommand(lookupSql, connection, transaction); + lookup.Parameters.AddWithValue("pluginId", request.PluginId); + lookup.Parameters.AddWithValue("schema", request.SchemaName); + using var reader = await lookup.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + Guid? existingPluginId = null; + string? existingSchema = null; + if (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + existingPluginId = reader.GetGuid(0); + existingSchema = reader.GetString(1); + } + + await reader.CloseAsync().ConfigureAwait(false); + + if (existingPluginId is not null + && (existingPluginId != request.PluginId || !string.Equals(existingSchema, request.SchemaName, StringComparison.Ordinal))) + { + throw new InvalidOperationException($"Schema '{request.SchemaName}' is already bound to another plugin registration."); + } + + if (existingPluginId is null && existed && !request.AdoptExistingSchema) + { + throw new InvalidOperationException($"Schema '{request.SchemaName}' already exists and must be explicitly adopted."); + } + + var pluginName = request.SchemaName[..request.SchemaName.LastIndexOf('_')]; + const string upsertSql = "INSERT INTO jellyfin_provider.plugin_schemas (plugin_id, plugin_name, schema_name, protocol_version) VALUES (@pluginId, @pluginName, @schema, 1) ON CONFLICT (plugin_id) DO UPDATE SET last_seen_at = now(), state = 'active';"; + using var upsert = new NpgsqlCommand(upsertSql, connection, transaction); + upsert.Parameters.AddWithValue("pluginId", request.PluginId); + upsert.Parameters.AddWithValue("pluginName", pluginName); + upsert.Parameters.AddWithValue("schema", request.SchemaName); + await upsert.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task SetSchemaAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, PluginSchema schema, CancellationToken cancellationToken) + { + using var command = new NpgsqlCommand($"SET LOCAL search_path TO {QuoteIdentifier(schema.Name)}, pg_catalog;", connection, transaction); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureMigrationTableAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, CancellationToken cancellationToken) + { + const string sql = "CREATE TABLE IF NOT EXISTS \"__pg_provider_schema_migrations\" (\"Id\" text PRIMARY KEY, \"Checksum\" text NOT NULL, \"AppliedAt\" timestamp with time zone NOT NULL DEFAULT now())"; + await ExecuteInTransactionAsync(connection, transaction, sql, null, cancellationToken).ConfigureAwait(false); + } + + private static async Task AcquireMigrationLockAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, PluginSchema schema, CancellationToken cancellationToken) + { + using var command = new NpgsqlCommand("SELECT pg_advisory_xact_lock(hashtext(@schema));", connection, transaction); + command.Parameters.AddWithValue("schema", schema.Name); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task GetAppliedChecksumAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, string id, CancellationToken cancellationToken) + { + using var command = new NpgsqlCommand($"SELECT \"Checksum\" FROM {QuoteIdentifier(MigrationTable)} WHERE \"Id\" = @id;", connection, transaction); + command.Parameters.AddWithValue("id", id); + return (string?)await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task RecordMigrationAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, PluginSchemaMigration migration, CancellationToken cancellationToken) + { + using var command = new NpgsqlCommand($"INSERT INTO {QuoteIdentifier(MigrationTable)} (\"Id\", \"Checksum\") VALUES (@id, @checksum);", connection, transaction); + command.Parameters.AddWithValue("id", migration.Id); + command.Parameters.AddWithValue("checksum", migration.Checksum); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task ExecuteInTransactionAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, string sql, IReadOnlyList? parameters, CancellationToken cancellationToken) + { + using var command = CreateCommand(connection, transaction, sql, parameters); + return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private static NpgsqlCommand CreateCommand(NpgsqlConnection connection, NpgsqlTransaction transaction, string sql, IReadOnlyList? parameters) + { + var command = new NpgsqlCommand(sql, connection, transaction); + if (parameters is null) + { + return command; + } + + foreach (var parameter in parameters) + { + ArgumentNullException.ThrowIfNull(parameter); + if (!ParameterNameRegex().IsMatch(parameter.Name)) + { + command.Dispose(); + throw new ArgumentException($"Invalid parameter name '{parameter.Name}'.", nameof(parameters)); + } + + command.Parameters.AddWithValue(parameter.Name, parameter.Value ?? DBNull.Value); + } + + return command; + } + + private async Task OpenConnectionAsync(CancellationToken cancellationToken) + { + var configured = _connectionStringFactory(); + if (string.IsNullOrWhiteSpace(configured)) + { + throw new InvalidOperationException("PostgreSQL is not the active Jellyfin database provider."); + } + + var builder = PostgresDatabaseProvider.BuildTunedConnectionString( + configured, + PostgresPlugin.Instance?.Configuration, + PostgresPlugin.Instance?.Configuration.CommandTimeout ?? 600); + var connection = new NpgsqlConnection(builder.ConnectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + return connection; + } + + private static void ValidateSchema(PluginSchema schema) + { + if (schema.PluginId == Guid.Empty) + { + throw new ArgumentException("The schema handle has no plugin owner.", nameof(schema)); + } + + ValidateSchemaName(schema.Name); + } + + private static void ValidateSchemaName(string schemaName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(schemaName); + if (!SchemaNameRegex().IsMatch(schemaName) || string.Equals(schemaName, "public", StringComparison.Ordinal) || !SchemaGuidSuffixRegex().IsMatch(schemaName)) + { + throw new ArgumentException("Schema names must use lowercase _ format and cannot be public.", nameof(schemaName)); + } + } + + private static void ValidateMigrations(IReadOnlyList migrations) + { + var seen = new HashSet(StringComparer.Ordinal); + foreach (var migration in migrations) + { + ArgumentNullException.ThrowIfNull(migration); + if (string.IsNullOrWhiteSpace(migration.Id) || migration.Id.Length > 128 || !seen.Add(migration.Id)) + { + throw new ArgumentException("Migration identifiers must be unique, non-empty, and at most 128 characters.", nameof(migrations)); + } + + ValidateDdlSql(migration.Sql); + } + } + + private static void ValidateDdlSql(string sql) => ValidateSql(sql, "CREATE", "ALTER", "DROP"); + + private static void ValidateDataSql(string sql) => ValidateSql(sql, "INSERT", "UPDATE", "DELETE"); + + private static void ValidateQuerySql(string sql) => ValidateSql(sql, "SELECT", "WITH"); + + private static void ValidateSql(string sql, params string[] allowedPrefixes) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sql); + var normalized = sql.Trim(); + if (normalized.EndsWith(';')) + { + normalized = normalized[..^1].TrimEnd(); + } + + if (normalized.Length == 0 || normalized.Length > MaxSqlLength || !allowedPrefixes.Any(prefix => normalized.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) + { + throw new ArgumentException("The SQL operation is not allowed by the private-schema contract.", nameof(sql)); + } + + if (normalized.Contains(';', StringComparison.Ordinal) || ForbiddenSqlRegex().IsMatch(normalized) || QualifiedIdentifierRegex().IsMatch(normalized)) + { + throw new ArgumentException("SQL must be one unqualified statement limited to the plugin's private schema.", nameof(sql)); + } + } + + private static string QuoteIdentifier(string identifier) => string.Concat("\"", identifier.Replace("\"", "\"\"", StringComparison.Ordinal), "\""); + + [GeneratedRegex("^[A-Za-z][A-Za-z0-9_]*$", RegexOptions.CultureInvariant)] + private static partial Regex ParameterNameRegex(); + + [GeneratedRegex("^[a-z][a-z0-9_]{0,62}$", RegexOptions.CultureInvariant)] + private static partial Regex SchemaNameRegex(); + + [GeneratedRegex("_[0-9a-f]{32}$", RegexOptions.CultureInvariant)] + private static partial Regex SchemaGuidSuffixRegex(); + + [GeneratedRegex("(?:--|/\\*|\\*/|\\b(?:public|information_schema|pg_catalog|set_config|dblink|copy|vacuum|analyze|grant|revoke|create\\s+(?:role|database|schema|extension)|alter\\s+role|drop\\s+schema)\\b)", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex ForbiddenSqlRegex(); + + [GeneratedRegex("(?:\\\"[A-Za-z_][A-Za-z0-9_]*\\\"|[A-Za-z_][A-Za-z0-9_]*)\\s*\\.", RegexOptions.CultureInvariant)] + private static partial Regex QualifiedIdentifierRegex(); +} diff --git a/Jellyfin.Database.Providers.Postgres/Services/PostgresRecommendationQueryProvider.cs b/Jellyfin.Database.Providers.Postgres/Services/PostgresRecommendationQueryProvider.cs deleted file mode 100644 index bee6c1f..0000000 --- a/Jellyfin.Database.Providers.Postgres/Services/PostgresRecommendationQueryProvider.cs +++ /dev/null @@ -1,354 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using Jellyfin.Plugin.JellyTrend.Api; -using Microsoft.Extensions.Logging; -using Npgsql; - -namespace Jellyfin.Database.Providers.Postgres.Services; - -/// -/// Optimised PostgreSQL implementation of . -/// Uses Npgsql directly (bypassing EF Core) to issue single-pass queries with native -/// PostgreSQL array operators (&&) and GIN-indexed lookups, replacing the -/// correlated-subquery pattern generated by EF Core that caused ~45 s response times. -/// -/// -/// -/// Genres, Tags and Studios are stored pipe-delimited (|) in BaseItems. -/// We convert them to arrays with string_to_array so the && -/// (overlap) operator can use GIN indexes. -/// -/// -/// This class is registered only when the Postgres plugin is active. If the user -/// has not installed JellyTrend, Jellyfin's DI container will not resolve -/// and this class is never instantiated. -/// -/// -public sealed class PostgresRecommendationQueryProvider : IRecommendationQueryProvider -{ - /// - /// Filtro de pelicula sobre BaseItems."Type". - /// - /// - /// Jellyfin guarda ahi el nombre del tipo CLR (el discriminador de EF), - /// MediaBrowser.Controller.Entities.Movies.Movie, no "Movie": filtrar por el nombre corto no - /// devuelve ninguna fila. Se aceptan las dos formas para no depender de esa convencion. - /// - private const string MovieTypeFilter = "(b.\"Type\" = 'Movie' OR b.\"Type\" LIKE '%.Movie')"; - - private readonly ILogger _logger; - - /// - /// Initializes a new instance of the class. - /// - /// Logger for query diagnostics. - public PostgresRecommendationQueryProvider(ILogger logger) - { - _logger = logger; - } - - private static NpgsqlConnection OpenConnection() - { - var connStr = PostgresPlugin.Instance?.Configuration?.ConnectionString - ?? throw new InvalidOperationException("PostgreSQL connection string is not configured."); - var conn = new NpgsqlConnection(connStr); - conn.Open(); - return conn; - } - - /// - public IReadOnlyList GetPlayedMovies(Guid userId, int limit) - { - const string Sql = $""" - SELECT b."Id", b."Genres", b."Tags", b."Studios", b."CommunityRating", b."PremiereDate", - p."ProviderValue" AS "TmdbId" - FROM "BaseItems" b - LEFT JOIN "BaseItemProviders" p ON p."ItemId" = b."Id" AND p."ProviderId" = 'Tmdb' - WHERE {MovieTypeFilter} - AND b."IsVirtualItem" = false - AND EXISTS ( - SELECT 1 FROM "UserData" ud - WHERE ud."ItemId" = b."Id" - AND ud."UserId" = @userId - AND ud."Played" = true - ) - ORDER BY b."DateModified" DESC - LIMIT @limit - """; - - return ExecuteQuery(Sql, userId, limit); - } - - /// - public IReadOnlyList GetResumableMovies(Guid userId, int limit) - { - const string Sql = $""" - SELECT b."Id", b."Genres", b."Tags", b."Studios", b."CommunityRating", b."PremiereDate", - p."ProviderValue" AS "TmdbId" - FROM "BaseItems" b - LEFT JOIN "BaseItemProviders" p ON p."ItemId" = b."Id" AND p."ProviderId" = 'Tmdb' - WHERE {MovieTypeFilter} - AND b."IsVirtualItem" = false - AND EXISTS ( - SELECT 1 FROM "UserData" ud - WHERE ud."ItemId" = b."Id" - AND ud."UserId" = @userId - AND ud."PlaybackPositionTicks" > 0 - AND ud."Played" = false - ) - LIMIT @limit - """; - - return ExecuteQuery(Sql, userId, limit); - } - - /// - public IReadOnlyList GetUnwatchedMoviesByGenres( - Guid userId, - IReadOnlyList genres, - IReadOnlyList topParentIds, - int limit) - { - if (genres.Count == 0) - { - return []; - } - - var sql = BuildUnwatchedSql( - extraWhere: """ - AND string_to_array(b."Genres", '|') && @facets - """, - hasTopParents: topParentIds.Count > 0); - - return ExecuteUnwatchedQuery(sql, userId, genres.ToArray(), topParentIds, limit); - } - - /// - public IReadOnlyList GetUnwatchedMoviesByPersons( - Guid userId, - IReadOnlyList personIds, - IReadOnlyList topParentIds, - int limit) - { - if (personIds.Count == 0) - { - return []; - } - - // PeopleBaseItemMap joins on ItemId; filter by any of the given person ids. - var sql = BuildUnwatchedSql( - extraWhere: """ - AND EXISTS ( - SELECT 1 FROM "PeopleBaseItemMap" pm - WHERE pm."ItemId" = b."Id" - AND pm."PeopleId" = ANY(@facets) - ) - """, - hasTopParents: topParentIds.Count > 0, - facetType: NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Uuid); - - return ExecuteUnwatchedQuery(sql, userId, personIds.ToArray(), topParentIds, limit); - } - - /// - public IReadOnlyList GetUnwatchedMoviesByTags( - Guid userId, - IReadOnlyList tags, - IReadOnlyList topParentIds, - int limit) - { - if (tags.Count == 0) - { - return []; - } - - var sql = BuildUnwatchedSql( - extraWhere: """ - AND string_to_array(b."Tags", '|') && @facets - """, - hasTopParents: topParentIds.Count > 0); - - return ExecuteUnwatchedQuery(sql, userId, tags.ToArray(), topParentIds, limit); - } - - /// - public IReadOnlyList GetRandomUnwatchedMovies( - Guid userId, - IReadOnlyList topParentIds, - int limit) - { - var topParentFilter = topParentIds.Count > 0 - ? "AND b.\"TopParentId\" = ANY(@topParentIds)" - : string.Empty; - - var sql = $""" - SELECT b."Id", b."Genres", b."Tags", b."Studios", b."CommunityRating", b."PremiereDate", - p."ProviderValue" AS "TmdbId" - FROM "BaseItems" b - LEFT JOIN "BaseItemProviders" p ON p."ItemId" = b."Id" AND p."ProviderId" = 'Tmdb' - WHERE {MovieTypeFilter} - AND b."IsVirtualItem" = false - {topParentFilter} - AND NOT EXISTS ( - SELECT 1 FROM "UserData" ud - WHERE ud."ItemId" = b."Id" - AND ud."UserId" = @userId - AND (ud."Played" = true OR ud."PlaybackPositionTicks" > 0) - ) - ORDER BY random() - LIMIT @limit - """; - - try - { - using var conn = OpenConnection(); - using var cmd = conn.CreateCommand(); - cmd.CommandText = sql; - cmd.Parameters.AddWithValue("userId", userId); - if (topParentIds.Count > 0) - { - cmd.Parameters.Add(new NpgsqlParameter("topParentIds", NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Uuid) - { - Value = topParentIds.ToArray() - }); - } - - cmd.Parameters.AddWithValue("limit", limit); - return ReadItems(cmd); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "[JellyTrend] GetRandomUnwatchedMovies failed; returning empty list"); - return []; - } - } - - // ────────────────────────────────────────────────────────────────────────────── - // Helpers - // ────────────────────────────────────────────────────────────────────────────── - - private static string BuildUnwatchedSql( - string extraWhere, - bool hasTopParents, - NpgsqlTypes.NpgsqlDbType facetType = NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Text) - { - _ = facetType; // consumed by caller via the overloaded ExecuteUnwatchedQuery - var topParentFilter = hasTopParents - ? "AND b.\"TopParentId\" = ANY(@topParentIds)" - : string.Empty; - - return $""" - SELECT b."Id", b."Genres", b."Tags", b."Studios", b."CommunityRating", b."PremiereDate", - p."ProviderValue" AS "TmdbId" - FROM "BaseItems" b - LEFT JOIN "BaseItemProviders" p ON p."ItemId" = b."Id" AND p."ProviderId" = 'Tmdb' - WHERE {MovieTypeFilter} - AND b."IsVirtualItem" = false - {topParentFilter} - {extraWhere} - AND NOT EXISTS ( - SELECT 1 FROM "UserData" ud - WHERE ud."ItemId" = b."Id" - AND ud."UserId" = @userId - AND (ud."Played" = true OR ud."PlaybackPositionTicks" > 0) - ) - ORDER BY b."CommunityRating" DESC NULLS LAST - LIMIT @limit - """; - } - - [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "SQL is a hardcoded constant string — no user input reaches CommandText.")] - private List ExecuteQuery(string sql, Guid userId, int limit) - { - try - { - using var conn = OpenConnection(); - using var cmd = conn.CreateCommand(); - cmd.CommandText = sql; - cmd.Parameters.AddWithValue("userId", userId); - cmd.Parameters.AddWithValue("limit", limit); - return ReadItems(cmd); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "[JellyTrend] Query failed; returning empty list. SQL: {Sql}", sql); - return []; - } - } - - [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "SQL is built from hardcoded templates only; no user input reaches CommandText.")] - private List ExecuteUnwatchedQuery( - string sql, - Guid userId, - Array facets, - IReadOnlyList topParentIds, - int limit) - { - try - { - using var conn = OpenConnection(); - using var cmd = conn.CreateCommand(); - cmd.CommandText = sql; - cmd.Parameters.AddWithValue("userId", userId); - - // Facet parameter type depends on the caller (string[] or Guid[]). - if (facets is string[] strFacets) - { - cmd.Parameters.Add(new NpgsqlParameter("facets", NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Text) { Value = strFacets }); - } - else if (facets is Guid[] guidFacets) - { - cmd.Parameters.Add(new NpgsqlParameter("facets", NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Uuid) { Value = guidFacets }); - } - - if (topParentIds.Count > 0) - { - cmd.Parameters.Add(new NpgsqlParameter("topParentIds", NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Uuid) - { - Value = topParentIds.ToArray() - }); - } - - cmd.Parameters.AddWithValue("limit", limit); - return ReadItems(cmd); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "[JellyTrend] Unwatched query failed; returning empty list. SQL: {Sql}", sql); - return []; - } - } - - private static List ReadItems(NpgsqlCommand cmd) - { - var results = new List(); - using var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess); - while (reader.Read()) - { - var id = reader.GetGuid(0); - var genres = SplitPipeDelimited(reader.IsDBNull(1) ? null : reader.GetString(1)); - var tags = SplitPipeDelimited(reader.IsDBNull(2) ? null : reader.GetString(2)); - var studios = SplitPipeDelimited(reader.IsDBNull(3) ? null : reader.GetString(3)); - float? rating = reader.IsDBNull(4) ? null : reader.GetFloat(4); - DateTime? premiereDate = reader.IsDBNull(5) ? null : reader.GetDateTime(5).ToUniversalTime(); - string? tmdbId = reader.IsDBNull(6) ? null : reader.GetString(6); - - results.Add(new RecommendationItem(id, tmdbId, genres, tags, studios, rating, premiereDate)); - } - - return results; - } - - private static string[] SplitPipeDelimited(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return []; - } - - return value.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - } -} diff --git a/README.en.md b/README.en.md index 08e627c..f971731 100644 --- a/README.en.md +++ b/README.en.md @@ -50,7 +50,7 @@ automatic health check, scheduled maintenance and optional JellyTrend integratio - 🛡️ **Error prevention** — EF Core interceptors for upserts, DB error logging and `DateTime.Kind` normalization. - 🔧 **Automatic optimization** — GIN indexes CONCURRENTLY, autovacuum tuning on critical tables. - 💾 **Scheduled backups** with `pg_dump`, optional ZIP compression and restore from the UI. -- ⚡ **Optional JellyTrend integration** — with JellyTrend 3.x its data lives in the `jellytrend` schema of your database (created only when that plugin is present) and recommendations are **4-10× faster** with optimized native SQL. +- 🔐 **Private-schema host** — a reusable contract through which other plugins persist only their own data, without credentials or access to Jellyfin's `public` schema. - 📊 **Database statistics** — total size, active connections and per-table metrics from the UI. - 🧪 **Read-only SQL console** — run diagnostic queries from the panel with 10 templates and history; any data-modifying statement is rejected. - 🔁 **One-click rollback to SQLite** from the configuration tab. @@ -298,31 +298,6 @@ In **Maintenance**: **Create backup now** / **Restore backup** (accepts `.sql` o --- -## ⚡ JellyTrend integration - -If you have [**JellyTrend**](https://github.com/BORNIOS/JellyTrend) installed, this plugin offers it **two things** and JellyTrend detects both on its own, with nothing to configure: - -| Role | What it does | -|---|---| -| 🗄️ **Data store** (JellyTrend 3.x) | JellyTrend creates the **`jellytrend`** schema in your database and keeps its feature cache, trending list, per-user taste profiles and recommendations, hidden titles and run history there. The schema is created the first time JellyTrend asks for it: **a server without JellyTrend never sees those tables** | -| 🔍 **Query accelerator** | The recommendation engine replaces `ILibraryManager` queries with native SQL using the `&&` array operator and the GIN indexes — **4-10× faster** | - -| Engine | Behavior | -|---|---| -| **SQLite** (without this plugin) | `ILibraryManager` — compatible with any installation | -| **PostgreSQL** (with this plugin) | Its own store + direct SQL with GIN indexes — **4-10× faster** | - -**About the `jellytrend` schema:** - -- It lives **outside `public`** on purpose: the PostgreSQL to SQLite export enumerates the tables of `public`, so these **never end up inside a SQLite file**, and the Health Check and the panel statistics keep analysing Jellyfin's tables only. -- JellyTrend still writes its JSON files to `{DataDir}/data/JellyTrend` as a **courtesy copy**: removing either plugin loses nothing. -- Current schema version: **v4**, and JellyTrend shows it on its *Activity* tab. - -> ✅ No manual setup needed: if both plugins are installed, both integrations activate on their own. -> ℹ️ And it is optional in both directions: JellyTrend runs just as well on SQLite, and this provider runs without JellyTrend. - ---- - ## ✅ Post-migration checklist 1. Login works with existing users. diff --git a/README.md b/README.md index cc886be..fa919db 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ health check automático, mantenimiento programado e integración opcional con J - 🛡️ **Prevención de errores** — interceptores EF Core para upserts, logging de errores DB y normalización de `DateTime.Kind`. - 🔧 **Optimización automática** — índices GIN CONCURRENTLY, tuning de autovacuum en tablas críticas. - 💾 **Backups programados** con `pg_dump`, compresión ZIP opcional y restore desde la UI. -- ⚡ **Integración opcional con JellyTrend** — con JellyTrend 3.x sus datos viven en el esquema `jellytrend` de tu base (creado solo si ese plugin está) y las recomendaciones van **4-10× más rápidas** con SQL nativo optimizado. +- 🔐 **Host de esquemas privados** — contrato reutilizable para que otros plugins persistan únicamente sus propios datos, sin credenciales ni acceso al esquema `public` de Jellyfin. - 📊 **Estadísticas de BD** — tamaño, conexiones activas y análisis tabla por tabla desde la UI. - 🧪 **Consola SQL de solo lectura** — lanza consultas de diagnóstico desde el panel con 10 plantillas e historial; rechaza cualquier sentencia que modifique datos. - 🔁 **Rollback a SQLite** en un clic desde la configuración. @@ -301,31 +301,6 @@ En **Mantenimiento**: **Crear backup ahora** / **Restablecer backup** (acepta `. --- -## ⚡ Integración con JellyTrend - -Si tienes instalado [**JellyTrend**](https://github.com/BORNIOS/JellyTrend), este plugin le ofrece **dos cosas** y JellyTrend las detecta solo, sin configurar nada: - -| Rol | Qué hace | -|---|---| -| 🗄️ **Almacén de datos** (JellyTrend 3.x) | JellyTrend crea el esquema **`jellytrend`** en tu base y guarda ahí su caché de características, la lista de tendencias, los perfiles de gusto y las recomendaciones de cada usuario, los títulos ocultos y el historial de corridas. El esquema se crea la primera vez que JellyTrend lo pide: **un servidor sin JellyTrend nunca ve esas tablas** | -| 🔍 **Acelerador de consultas** | El motor de recomendaciones sustituye las consultas de `ILibraryManager` por SQL nativo con el operador `&&` de arrays y los índices GIN — **4-10× más rápido** | - -| Motor | Comportamiento | -|---|---| -| **SQLite** (sin este plugin) | `ILibraryManager` — compatible con cualquier instalación | -| **PostgreSQL** (con este plugin) | Almacén propio + SQL directo con índices GIN — **4-10× más rápido** | - -**Sobre el esquema `jellytrend`:** - -- Vive **fuera de `public`** a propósito: la exportación de PostgreSQL a SQLite enumera las tablas de `public`, así que estas **nunca acaban dentro de un archivo SQLite**, y el Health Check y las estadísticas del panel siguen analizando solo las tablas de Jellyfin. -- JellyTrend sigue escribiendo sus JSON en `{DataDir}/data/JellyTrend` como **copia de cortesía**: quitar cualquiera de los dos plugins no pierde nada. -- Versión actual del esquema: **v4**, y JellyTrend la muestra en su pestaña *Actividad*. - -> ✅ Sin configuración manual: si ambos plugins están instalados, las dos integraciones se activan solas. -> ℹ️ Y es opcional en los dos sentidos: JellyTrend funciona igual sobre SQLite, y este proveedor funciona sin JellyTrend. - ---- - ## ✅ Checklist post-migración 1. Login correcto con los usuarios existentes. diff --git a/Tests/Jellyfin.Database.Providers.Postgres.Tests/HomeQueryCacheTests.cs b/Tests/Jellyfin.Database.Providers.Postgres.Tests/HomeQueryCacheTests.cs index 1c7f580..6d62a63 100644 --- a/Tests/Jellyfin.Database.Providers.Postgres.Tests/HomeQueryCacheTests.cs +++ b/Tests/Jellyfin.Database.Providers.Postgres.Tests/HomeQueryCacheTests.cs @@ -131,11 +131,50 @@ await ExecuteAsync( } /// - /// A cold home-query cache must execute one database command even when many callers arrive together. - /// The sequence exposes physical executions: each execution would otherwise return a different value. + /// A deterministic 100-row BaseItems page must warm only its immediate successor. The request for the + /// second page is then served from memory, while that cache hit starts warming page three. /// [SkippableFact] - public async Task ConcurrentMissesForTheSameKeyShareOneDatabaseExecution() + public async Task FullBaseItemsPagePrefetchesTheNextPage() + { + Skip.IfNot(TestEnvironment.PostgresAvailable, SkipReason); + await using var scratch = await TestEnvironment.ScratchDatabaseScope.CreateAsync(); + HomeQueryCacheInterceptor.Purge(); + await MigrateAsync(scratch.Database.ConnectionString); + + await ExecuteAsync( + scratch.Database.ConnectionString, + """ + INSERT INTO "BaseItems" ("Id", "Type", "IsMovie", "IsLocked", "IsSeries", "IsRepeat", "IsInMixedFolder", "IsFolder", "IsVirtualItem", "Name") + SELECT md5('home-prefetch-' || value::text)::uuid, 'Movie', false, false, false, false, false, false, false, 'Home prefetch ' || lpad(value::text, 3, '0') + FROM generate_series(0, 200) AS value; + """); + + var interceptor = new HomeQueryCacheInterceptor(); + using var context = CreateContext(scratch.Database.ConnectionString, interceptor); + const string Sql = "SELECT \"Id\" AS \"Value\" FROM \"BaseItems\" ORDER BY \"Name\" LIMIT @limit OFFSET @offset"; + + var firstPage = await context.Database.SqlQueryRaw(Sql, new NpgsqlParameter { ParameterName = "limit", Value = 100 }, new NpgsqlParameter { ParameterName = "offset", Value = 0 }).ToListAsync(); + Assert.Equal(100, firstPage.Count); + + for (var attempt = 0; attempt < 50 && HomeQueryCacheInterceptor.PrefetchCompletedCount == 0; attempt++) + { + await Task.Delay(50); + } + + Assert.Equal(1, HomeQueryCacheInterceptor.PrefetchCompletedCount); + + var secondPage = await context.Database.SqlQueryRaw(Sql, new NpgsqlParameter { ParameterName = "limit", Value = 100 }, new NpgsqlParameter { ParameterName = "offset", Value = 100 }).ToListAsync(); + Assert.Equal(100, secondPage.Count); + Assert.Equal(1, HomeQueryCacheInterceptor.PrefetchedPageHitCount); + } + + /// + /// Media-table queries must retain Npgsql's native reader conversions. A DataTableReader used to turn + /// a numeric expression into Int32 and make EF Core's GetFloat fail in the built-in search provider. + /// + [SkippableFact] + public async Task CachedMediaTableQueryPreservesSingle() { Skip.IfNot(TestEnvironment.PostgresAvailable, SkipReason); await using var scratch = await TestEnvironment.ScratchDatabaseScope.CreateAsync(); @@ -146,7 +185,6 @@ public async Task ConcurrentMissesForTheSameKeyShareOneDatabaseExecution() await ExecuteAsync( scratch.Database.ConnectionString, string.Concat( - "CREATE SEQUENCE home_query_cache_probe; ", "INSERT INTO \"ItemValues\" (\"ItemValueId\", \"Type\", \"Value\", \"CleanValue\") VALUES ('", itemId.ToString(), "', 2, 'SingleFlightProbe', 'singleflightprobe');")); @@ -158,14 +196,14 @@ await ExecuteAsync( await start.Task; using var context = CreateContext(scratch.Database.ConnectionString, interceptor); return await context.Database - .SqlQueryRaw("SELECT nextval('home_query_cache_probe') AS \"Value\" FROM \"ItemValues\" LIMIT 1") + .SqlQueryRaw("SELECT 1::real AS \"Value\" FROM \"ItemValues\" LIMIT 1") .SingleAsync(); }); start.SetResult(); var results = await Task.WhenAll(callers); - Assert.All(results, value => Assert.Equal(1, value)); + Assert.All(results, value => Assert.Equal(1f, value)); Assert.Equal(0, HomeQueryCacheInterceptor.InFlightCount); } @@ -195,6 +233,10 @@ private static async Task MigrateAsync(string connectionString) private static JellyfinDbContext CreateContext(string connectionString, params IInterceptor[] interceptors) { + connectionString = new NpgsqlConnectionStringBuilder(connectionString) + { + PersistSecurityInfo = true, + }.ConnectionString; var builder = new DbContextOptionsBuilder() .UseNpgsql(connectionString, pg => pg.MigrationsAssembly(typeof(PostgresDatabaseProvider).Assembly.FullName)); if (interceptors.Length > 0) diff --git a/Tests/Jellyfin.Database.Providers.Postgres.Tests/JellyTrendContractTests.cs b/Tests/Jellyfin.Database.Providers.Postgres.Tests/JellyTrendContractTests.cs deleted file mode 100644 index 64d320d..0000000 --- a/Tests/Jellyfin.Database.Providers.Postgres.Tests/JellyTrendContractTests.cs +++ /dev/null @@ -1,107 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; - -using Jellyfin.Database.Providers.Postgres.Services; -using Jellyfin.Plugin.JellyTrend.Api; -using Microsoft.Extensions.Logging; -using Xunit; - -namespace Jellyfin.Database.Providers.Postgres.Tests; - -/// -/// Fija la parte del contrato con JellyTrend que el compilador no puede comprobar. -/// -/// -/// -/// Este plugin compila su propia copia del contrato (JellyTrendContract/) y JellyTrend lo encuentra -/// buscando esa copia entre los ensamblados cargados, enlazando cada llamada por nombre. Es decir: los -/// nombres SON el contrato. Renombrar un metodo, cambiar un parametro o tocar las propiedades de la -/// proyeccion rompe la integracion sin error de compilacion en ninguno de los dos lados. -/// -/// -/// Estas pruebas, junto con su gemela en el repo de JellyTrend, obligan a que las dos versiones avancen -/// alineadas. La dependencia sigue siendo opcional en tiempo de ejecucion: sin JellyTrend instalado no se -/// pierde ninguna funcionalidad de este plugin, y sin este plugin JellyTrend usa ILibraryManager. -/// -/// -public sealed class JellyTrendContractTests -{ - /// JellyTrend reconoce esta copia por su nombre completo, no por el nombre del plugin. - [Fact] - public void ElEspejoConservaElNombreCompletoDelContrato() - { - Assert.Equal("Jellyfin.Plugin.JellyTrend.Api.IRecommendationQueryProvider", typeof(IRecommendationQueryProvider).FullName); - Assert.Equal("Jellyfin.Plugin.JellyTrend.Api.RecommendationItem", typeof(RecommendationItem).FullName); - } - - [Fact] - public void ElEspejoTieneLaMismaFormaQueElContratoDeJellyTrend() - { - Assert.Equal( - [ - "GetPlayedMovies(Guid, Int32) -> IReadOnlyList", - "GetRandomUnwatchedMovies(Guid, IReadOnlyList, Int32) -> IReadOnlyList", - "GetResumableMovies(Guid, Int32) -> IReadOnlyList", - "GetUnwatchedMoviesByGenres(Guid, IReadOnlyList, IReadOnlyList, Int32) -> IReadOnlyList", - "GetUnwatchedMoviesByPersons(Guid, IReadOnlyList, IReadOnlyList, Int32) -> IReadOnlyList", - "GetUnwatchedMoviesByTags(Guid, IReadOnlyList, IReadOnlyList, Int32) -> IReadOnlyList" - ], - typeof(IRecommendationQueryProvider).GetMethods().Select(Describe).OrderBy(static text => text, StringComparer.Ordinal)); - } - - /// - /// La busqueda de JellyTrend lee la proyeccion por nombre de propiedad: si cambian, deja de entenderla. - /// - [Fact] - public void LaProyeccionTieneLasMismasPropiedades() - { - Assert.Equal( - [ - "CommunityRating: Nullable", - "Genres: IReadOnlyList", - "Id: Guid", - "PremiereDate: Nullable", - "Studios: IReadOnlyList", - "Tags: IReadOnlyList", - "TmdbId: String" - ], - typeof(RecommendationItem).GetProperties().Select(static property => $"{property.Name}: {Friendly(property.PropertyType)}").OrderBy(static text => text, StringComparer.Ordinal)); - } - - /// - /// JellyTrend construye la implementacion por reflexion: necesita un tipo publico y un constructor sin - /// parametros o con un unico , como el resto de servicios de Jellyfin. - /// - [Fact] - public void JellyTrendPuedeConstruirLaImplementacion() - { - var implementation = typeof(PostgresRecommendationQueryProvider); - - Assert.True(implementation.IsPublic, "El tipo debe ser publico para que se pueda encontrar."); - Assert.False(implementation.IsAbstract); - - var constructible = implementation.GetConstructors().Any(static constructor => - constructor.GetParameters().Length == 0 - || (constructor.GetParameters() is [{ ParameterType: var parameter }] - && parameter.IsGenericType - && parameter.GetGenericTypeDefinition() == typeof(ILogger<>))); - - Assert.True(constructible, "JellyTrend solo puede construir tipos sin parametros o con un unico ILogger."); - } - - private static string Describe(MethodInfo method) - => $"{method.Name}({string.Join(", ", method.GetParameters().Select(static parameter => Friendly(parameter.ParameterType)))}) -> {Friendly(method.ReturnType)}"; - - private static string Friendly(Type type) - { - if (type.IsGenericType && type.Name.Contains('`', StringComparison.Ordinal)) - { - var name = type.Name[..type.Name.IndexOf('`', StringComparison.Ordinal)]; - return $"{name}<{string.Join(", ", type.GetGenericArguments().Select(Friendly))}>"; - } - - return type.Name; - } -} diff --git a/Tests/Jellyfin.Database.Providers.Postgres.Tests/PluginSchemaHostTests.cs b/Tests/Jellyfin.Database.Providers.Postgres.Tests/PluginSchemaHostTests.cs new file mode 100644 index 0000000..602b294 --- /dev/null +++ b/Tests/Jellyfin.Database.Providers.Postgres.Tests/PluginSchemaHostTests.cs @@ -0,0 +1,88 @@ +using System.Data.Common; +using Jellyfin.Database.Providers.Postgres.Api; +using Jellyfin.Database.Providers.Postgres.Logging; +using Jellyfin.Database.Providers.Postgres.Services; +using Npgsql; +using Xunit; + +namespace Jellyfin.Database.Providers.Postgres.Tests; + +/// Integration coverage for the isolated private-schema contract. +public sealed class PluginSchemaHostTests +{ + private const string LiveSchemaProbeVariable = "POSTGRES_LIVE_SCHEMA_PROBE"; + + [SkippableFact] + public async Task PrivateSchemaSupportsMigrationsAndParameterizedDataWithoutPublicAccess() + { + Skip.IfNot(TestEnvironment.PostgresAvailable, "POSTGRES_TEST_CONNECTION is not set."); + await using var scratch = await TestEnvironment.ScratchDatabaseScope.CreateAsync(); + var host = new PostgresPluginSchemaHost(() => scratch.Database.ConnectionString); + var schema = await host.OpenSchemaAsync(Guid.Parse("eb587e52-bf22-48d0-ae9c-dc99f6d8f1dc")); + + await host.ApplyMigrationsAsync( + schema, + [new PluginSchemaMigration("001_create_state", "CREATE TABLE state (id integer PRIMARY KEY, value text NOT NULL)")]); + + await host.ExecuteAsync( + schema, + "INSERT INTO state (id, value) VALUES (@id, @value)", + [new PluginSqlParameter("id", 1), new PluginSqlParameter("value", "private")]); + + var value = await host.QueryAsync( + schema, + "SELECT value FROM state WHERE id = @id", + ReadSingleString, + [new PluginSqlParameter("id", 1)]); + + Assert.Equal("private", value); + await Assert.ThrowsAsync(() => host.QueryAsync(schema, "SELECT * FROM public.\"BaseItems\"", ReadSingleString)); + } + + [Fact] + public void SchemaNameIsStableAndDoesNotUsePublic() + { + var pluginId = Guid.Parse("eb587e52-bf22-48d0-ae9c-dc99f6d8f1dc"); + var first = new PluginSchema(pluginId, "pgp_eb587e52bf2248d0ae9cdc99f6d8f1dc"); + var second = new PluginSchema(pluginId, "pgp_eb587e52bf2248d0ae9cdc99f6d8f1dc"); + + Assert.Equal(first.Name, second.Name); + Assert.NotEqual("public", first.Name); + } + + /// + /// Opt-in probe for a running Jellyfin database. It records the provider's private-schema log entry + /// and removes its generated schema in a finally block, so it cannot affect Jellyfin's public schema. + /// + [SkippableFact] + public async Task LiveSchemaProbeCreatesAndRemovesOnlyItsGeneratedSchema() + { + Skip.IfNot(Environment.GetEnvironmentVariable(LiveSchemaProbeVariable) == "1", $"{LiveSchemaProbeVariable} is not set to 1."); + Skip.IfNot(TestEnvironment.PostgresAvailable, "POSTGRES_TEST_CONNECTION is not set."); + + var logDirectory = Environment.GetEnvironmentVariable("JELLYFIN_LOG_DIRECTORY"); + Skip.If(string.IsNullOrWhiteSpace(logDirectory), "JELLYFIN_LOG_DIRECTORY is not set."); + PostgresLog.SetLogDirectory(logDirectory); + + var host = new PostgresPluginSchemaHost(() => TestEnvironment.PostgresConnection); + var schema = await host.OpenSchemaAsync(Guid.NewGuid()); + try + { + Assert.StartsWith("pgp_", schema.Name, StringComparison.Ordinal); + Assert.NotEqual("public", schema.Name); + } + finally + { + await using var connection = new NpgsqlConnection(TestEnvironment.PostgresConnection); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand($"DROP SCHEMA IF EXISTS \"{schema.Name}\" CASCADE;", connection); + await command.ExecuteNonQueryAsync(); + } + } + + private static string ReadSingleString(DbDataReader reader) + { + Assert.True(reader.Read()); + return reader.GetString(0); + } +} diff --git a/Tests/Jellyfin.Database.Providers.Postgres.Tests/ProviderContractTests.cs b/Tests/Jellyfin.Database.Providers.Postgres.Tests/ProviderContractTests.cs index c244362..4cec5f6 100644 --- a/Tests/Jellyfin.Database.Providers.Postgres.Tests/ProviderContractTests.cs +++ b/Tests/Jellyfin.Database.Providers.Postgres.Tests/ProviderContractTests.cs @@ -1,9 +1,12 @@ using System.Reflection; using System.Text.RegularExpressions; using Jellyfin.Database.Implementations; +using Jellyfin.Database.Providers.Postgres.Api; +using Jellyfin.Database.Providers.Postgres.Services; using MediaBrowser.Common.Plugins; using MediaBrowser.Controller.Plugins; using MediaBrowser.Model.Plugins; +using Microsoft.Extensions.DependencyInjection; using Xunit; namespace Jellyfin.Database.Providers.Postgres.Tests; @@ -42,6 +45,20 @@ public void EntryPointsMatchWhatJellyfinInstantiates() Assert.True(typeof(IHasWebPages).IsAssignableFrom(typeof(PostgresPlugin))); } + /// + /// The optional cross-plugin contract is registered as a private-schema host, never as a raw database + /// connection or a provider for Jellyfin's own model. + /// + [Fact] + public void PrivateSchemaHostIsRegistered() + { + var services = new ServiceCollection(); + new PostgresServiceRegistrator().RegisterServices(services, null!); + + var registration = Assert.Single(services, service => service.ServiceType == typeof(IPluginSchemaHost)); + Assert.Equal(typeof(PostgresPluginSchemaHost), registration.ImplementationType); + } + /// /// A member of left as throw new NotImplementedException() /// only fails when the server calls it: PurgeDatabase, for instance, is used when restoring a diff --git a/build.yaml b/build.yaml index eb6f567..b4dce89 100644 --- a/build.yaml +++ b/build.yaml @@ -1,7 +1,7 @@ --- name: "PostgreSQL Database Provider" guid: "a2b5f3e8-4c1d-4f7a-9e6b-8d0c2f1a3b5e" -version: "3.0.2.0" +version: "3.0.3.0" targetAbi: "12.1.0.0" framework: "net10.0" overview: "Use PostgreSQL as Jellyfin's database backend" diff --git a/manifest.json b/manifest.json index ee40389..4d55924 100644 --- a/manifest.json +++ b/manifest.json @@ -3,7 +3,7 @@ "guid": "a2b5f3e8-4c1d-4f7a-9e6b-8d0c2f1a3b5e", "name": "PostgreSQL Database Provider", "description": "Production-ready PostgreSQL backend for Jellyfin. Bidirectional SQLite migration, near-instant search, health diagnostics, read-only SQL console and scheduled maintenance — no core modifications required.", - "overview": "Reemplaza SQLite por PostgreSQL como motor de base de datos de Jellyfin. Incluye migración bidireccional SQLite↔PostgreSQL, búsqueda casi instantánea con índices GIN, health check automático al arrancar, consola SQL de solo lectura, mantenimiento programado (VACUUM/REINDEX/Backup) e integración opcional con JellyTrend.", + "overview": "Reemplaza SQLite por PostgreSQL como motor de base de datos de Jellyfin. Incluye migración bidireccional SQLite↔PostgreSQL, búsqueda casi instantánea con índices GIN, health check automático al arrancar, consola SQL de solo lectura, mantenimiento programado (VACUUM/REINDEX/Backup) y un host seguro de esquemas privados para otros plugins.", "owner": ".:BORNIOS:.", "category": "Database", "imageUrl": "https://raw.githubusercontent.com/BORNIOS/Jellyfin-Database-Providers-Postgres/main/Jellyfin.Database.Providers.Postgres/Resources/logo.png", From 02c5515dc9dfc19445b87e3dbd1cbcf9c87dd9c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Ch=C3=A1vez=20Bornios?= Date: Sun, 27 Sep 2026 21:37:06 -0700 Subject: [PATCH 02/12] feat: migrate pre-registry plugin schemas safely Allow the generic private-schema host to atomically claim only a matching unregistered predecessor schema, preserving data while registering the GUID-qualified target. --- .../Api/IPluginSchemaHost.cs | 16 ++++ .../Services/PostgresPluginSchemaHost.cs | 74 +++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/Jellyfin.Database.Providers.Postgres/Api/IPluginSchemaHost.cs b/Jellyfin.Database.Providers.Postgres/Api/IPluginSchemaHost.cs index d47ecd6..d1ae5c5 100644 --- a/Jellyfin.Database.Providers.Postgres/Api/IPluginSchemaHost.cs +++ b/Jellyfin.Database.Providers.Postgres/Api/IPluginSchemaHost.cs @@ -33,6 +33,22 @@ public interface IPluginSchemaHost /// An opaque handle for the plugin's schema. Task OpenSchemaAsync(Guid pluginId, CancellationToken cancellationToken = default); + /// + /// Claims the immediate pre-registry schema matching the target plugin name and moves it to the + /// target private-schema name. + /// + /// The target schema name and immutable plugin owner. + /// Cancellation token. + /// The handle of the registered target schema. + /// + /// The only legacy source accepted is the exact prefix of 's target + /// name: name_guid may claim only name. The target must not exist and the source must + /// be unregistered. This preserves the source tables by an atomic PostgreSQL schema rename. + /// + Task MigratePreRegistrySchemaAsync( + PluginSchemaRequest request, + CancellationToken cancellationToken = default); + /// /// Applies versioned DDL migrations in the schema represented by . /// diff --git a/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs b/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs index ef0a272..f77d563 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs @@ -85,6 +85,59 @@ public async Task OpenSchemaAsync(PluginSchemaRequest request, Can } } + /// + public async Task MigratePreRegistrySchemaAsync( + PluginSchemaRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + if (request.PluginId == Guid.Empty) + { + throw new ArgumentException("A plugin identifier is required.", nameof(request)); + } + + ValidateSchemaName(request.SchemaName); + var legacySchemaName = request.SchemaName[..request.SchemaName.LastIndexOf('_')]; + ValidateLegacySchemaName(legacySchemaName); + + using var connection = await OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + try + { + await EnsureRegistryAsync(connection, transaction, cancellationToken).ConfigureAwait(false); + if (await SchemaExistsAsync(connection, transaction, request.SchemaName, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException($"Target schema '{request.SchemaName}' already exists; it will not be overwritten by a pre-registry migration."); + } + + if (!await SchemaExistsAsync(connection, transaction, legacySchemaName, cancellationToken).ConfigureAwait(false)) + { + throw new InvalidOperationException($"Pre-registry schema '{legacySchemaName}' does not exist."); + } + + await EnsureLegacySchemaIsUnregisteredAsync(connection, transaction, legacySchemaName, cancellationToken).ConfigureAwait(false); + using (var rename = new NpgsqlCommand($"ALTER SCHEMA {QuoteIdentifier(legacySchemaName)} RENAME TO {QuoteIdentifier(request.SchemaName)};", connection, transaction)) + { + await rename.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + await RegisterSchemaAsync( + connection, + transaction, + request with { AdoptExistingSchema = true }, + existed: true, + cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + PostgresLog.Info($"[PluginSchema] Esquema pre-registro migrado: {legacySchemaName} -> {request.SchemaName}."); + return new PluginSchema(request.PluginId, request.SchemaName); + } + catch + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + throw; + } + } + /// public async Task ApplyMigrationsAsync( PluginSchema schema, @@ -251,6 +304,17 @@ private static async Task RegisterSchemaAsync(NpgsqlConnection connection, Npgsq await upsert.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } + private static async Task EnsureLegacySchemaIsUnregisteredAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, string legacySchemaName, CancellationToken cancellationToken) + { + const string sql = "SELECT EXISTS (SELECT 1 FROM jellyfin_provider.plugin_schemas WHERE schema_name = @schema);"; + using var command = new NpgsqlCommand(sql, connection, transaction); + command.Parameters.AddWithValue("schema", legacySchemaName); + if ((bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!) + { + throw new InvalidOperationException($"Pre-registry schema '{legacySchemaName}' is already registered and cannot be claimed."); + } + } + private static async Task SetSchemaAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, PluginSchema schema, CancellationToken cancellationToken) { using var command = new NpgsqlCommand($"SET LOCAL search_path TO {QuoteIdentifier(schema.Name)}, pg_catalog;", connection, transaction); @@ -350,6 +414,16 @@ private static void ValidateSchemaName(string schemaName) } } + private static void ValidateLegacySchemaName(string schemaName) + { + if (!SchemaNameRegex().IsMatch(schemaName) + || string.Equals(schemaName, "public", StringComparison.Ordinal) + || string.Equals(schemaName, RegistrySchema, StringComparison.Ordinal)) + { + throw new ArgumentException("The derived pre-registry schema name is not eligible for migration.", nameof(schemaName)); + } + } + private static void ValidateMigrations(IReadOnlyList migrations) { var seen = new HashSet(StringComparer.Ordinal); From a6f378e5705471f5b98cdec899106e8cc636754a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Ch=C3=A1vez=20Bornios?= Date: Sun, 27 Sep 2026 21:59:08 -0700 Subject: [PATCH 03/12] feat: inspect registered plugin schemas in dashboard Add a bounded read-only schema explorer for registered plugin storage, including tables, columns and previews, while keeping public inaccessible through this surface. --- .../Controllers/PostgresController.cs | 91 ++++++ .../PostgresServiceRegistrator.cs | 1 + .../Services/Models/PluginSchemaColumnInfo.cs | 4 + .../Services/Models/PluginSchemaInfo.cs | 15 + .../Services/Models/PluginSchemaRows.cs | 6 + .../Services/Models/PluginSchemaTableInfo.cs | 4 + .../Services/PluginSchemaExplorerService.cs | 185 ++++++++++++ .../Services/PostgresPluginSchemaHost.cs | 8 +- .../Web/configurationPage.html | 268 ++++++++++++++++++ 9 files changed, 580 insertions(+), 2 deletions(-) create mode 100644 Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaColumnInfo.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaInfo.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaRows.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaTableInfo.cs create mode 100644 Jellyfin.Database.Providers.Postgres/Services/PluginSchemaExplorerService.cs diff --git a/Jellyfin.Database.Providers.Postgres/Controllers/PostgresController.cs b/Jellyfin.Database.Providers.Postgres/Controllers/PostgresController.cs index b4ebeaf..12bc251 100644 --- a/Jellyfin.Database.Providers.Postgres/Controllers/PostgresController.cs +++ b/Jellyfin.Database.Providers.Postgres/Controllers/PostgresController.cs @@ -39,6 +39,7 @@ public class PostgresController : ControllerBase private readonly MaintenanceService _maintenanceService; private readonly InstantSearchService _instantSearch; private readonly ExportToSqliteService _exportService; + private readonly PluginSchemaExplorerService _pluginSchemaExplorer; private readonly ILogger _logger; /// @@ -50,6 +51,7 @@ public class PostgresController : ControllerBase /// Maintenance service instance. /// Instant search service instance. /// Export service instance. + /// Read-only explorer for registered private plugin schemas. /// Logger instance. public PostgresController( IApplicationPaths appPaths, @@ -58,6 +60,7 @@ public PostgresController( MaintenanceService maintenanceService, InstantSearchService instantSearch, ExportToSqliteService exportService, + PluginSchemaExplorerService pluginSchemaExplorer, ILogger logger) { _appPaths = appPaths; @@ -66,6 +69,7 @@ public PostgresController( _maintenanceService = maintenanceService; _instantSearch = instantSearch; _exportService = exportService; + _pluginSchemaExplorer = pluginSchemaExplorer; _logger = logger; } @@ -1048,6 +1052,63 @@ await PostgresDatabaseProvider.RunOptimizationsAsync( return Accepted(new { Status = "Optimization job started in background." }); } + // ───────────────────────────────────────────────────────────────────────── + // Managed private plugin schemas + // ───────────────────────────────────────────────────────────────────────── + + /// Lists schemas registered by plugins through the private-schema host. + /// Cancellation token. + /// Registered schemas and compact storage statistics. + [HttpGet("PluginSchemas")] + [ProducesResponseType(StatusCodes.Status200OK)] + public async Task> GetPluginSchemas(CancellationToken cancellationToken) + { + var connectionString = GetActiveConnectionString(); + if (connectionString is null) + { + return NoActiveConnection(); + } + + var schemas = await _pluginSchemaExplorer.ListSchemasAsync(connectionString, cancellationToken).ConfigureAwait(false); + return Ok(new { Schemas = schemas }); + } + + /// Lists tables belonging to a registered private plugin schema. + /// Registered schema to inspect. + /// Cancellation token. + /// The schema tables. + [HttpGet("PluginSchemas/{schemaName}/Tables")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task> GetPluginSchemaTables(string schemaName, CancellationToken cancellationToken) + => await ExecuteSchemaInspectionAsync( + connectionString => _pluginSchemaExplorer.ListTablesAsync(connectionString, schemaName, cancellationToken)).ConfigureAwait(false); + + /// Lists columns belonging to a table in a registered private plugin schema. + /// Registered schema to inspect. + /// Table to inspect. + /// Cancellation token. + /// The table columns. + [HttpGet("PluginSchemas/{schemaName}/Tables/{tableName}/Columns")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task> GetPluginSchemaColumns(string schemaName, string tableName, CancellationToken cancellationToken) + => await ExecuteSchemaInspectionAsync( + connectionString => _pluginSchemaExplorer.ListColumnsAsync(connectionString, schemaName, tableName, cancellationToken)).ConfigureAwait(false); + + /// Returns a bounded read-only preview from a registered private plugin table. + /// Registered schema to inspect. + /// Table to preview. + /// Maximum rows to return. + /// Cancellation token. + /// Preview columns, rows and truncation indicator. + [HttpGet("PluginSchemas/{schemaName}/Tables/{tableName}/Rows")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task> GetPluginSchemaRows(string schemaName, string tableName, int limit, CancellationToken cancellationToken) + => await ExecuteSchemaInspectionAsync( + connectionString => _pluginSchemaExplorer.PreviewRowsAsync(connectionString, schemaName, tableName, limit, cancellationToken)).ConfigureAwait(false); + // ───────────────────────────────────────────────────────────────────────── // Read-only SQL console // ───────────────────────────────────────────────────────────────────────── @@ -1126,6 +1187,36 @@ public async Task> RunQuery( return string.IsNullOrWhiteSpace(configured) ? null : configured; } + private async Task> ExecuteSchemaInspectionAsync(Func> operation) + { + ArgumentNullException.ThrowIfNull(operation); + var connectionString = GetActiveConnectionString(); + if (connectionString is null) + { + return NoActiveConnection(); + } + + try + { + return Ok(await operation(connectionString).ConfigureAwait(false)); + } + catch (ArgumentException ex) + { + Logging.PostgresLog.Warn($"[PluginSchemas] Inspección rechazada: {ex.Message}"); + return BadRequest(new { Error = ex.Message }); + } + catch (Npgsql.PostgresException ex) + { + Logging.PostgresLog.Error($"[PluginSchemas] PostgreSQL rechazó la inspección ({ex.SqlState})", ex); + return BadRequest(new { Error = $"({ex.SqlState}) {ex.MessageText}" }); + } + catch (Exception ex) + { + Logging.PostgresLog.Error("[PluginSchemas] Error inspeccionando esquema privado", ex); + return BadRequest(new { Error = ex.Message }); + } + } + private ObjectResult NoActiveConnection() => StatusCode( StatusCodes.Status409Conflict, diff --git a/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs b/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs index 2e66b38..13efe28 100644 --- a/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs +++ b/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs @@ -29,6 +29,7 @@ public void RegisterServices(IServiceCollection serviceCollection, IServerApplic serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); diff --git a/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaColumnInfo.cs b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaColumnInfo.cs new file mode 100644 index 0000000..b76ce48 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaColumnInfo.cs @@ -0,0 +1,4 @@ +namespace Jellyfin.Database.Providers.Postgres.Services.Models; + +/// One column exposed for inspection in a registered private plugin schema. +public sealed record PluginSchemaColumnInfo(string ColumnName, string DataType, bool Nullable, int OrdinalPosition); diff --git a/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaInfo.cs b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaInfo.cs new file mode 100644 index 0000000..3bb6791 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaInfo.cs @@ -0,0 +1,15 @@ +using System; + +namespace Jellyfin.Database.Providers.Postgres.Services.Models; + +/// One private schema registered by a Jellyfin plugin. +public sealed record PluginSchemaInfo( + Guid PluginId, + string PluginName, + string SchemaName, + int ProtocolVersion, + DateTime CreatedAt, + DateTime LastSeenAt, + string State, + int TableCount, + string TotalSize); diff --git a/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaRows.cs b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaRows.cs new file mode 100644 index 0000000..ea48178 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaRows.cs @@ -0,0 +1,6 @@ +using System.Collections.Generic; + +namespace Jellyfin.Database.Providers.Postgres.Services.Models; + +/// A bounded, read-only preview of a private plugin table. +public sealed record PluginSchemaRows(IReadOnlyList Columns, IReadOnlyList> Rows, bool Truncated); diff --git a/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaTableInfo.cs b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaTableInfo.cs new file mode 100644 index 0000000..e21631a --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaTableInfo.cs @@ -0,0 +1,4 @@ +namespace Jellyfin.Database.Providers.Postgres.Services.Models; + +/// One table belonging to a registered private plugin schema. +public sealed record PluginSchemaTableInfo(string TableName, long EstimatedRows, string TotalSize); diff --git a/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaExplorerService.cs b/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaExplorerService.cs new file mode 100644 index 0000000..b761ae4 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaExplorerService.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Database.Providers.Postgres.Services.Models; +using Npgsql; + +namespace Jellyfin.Database.Providers.Postgres.Services; + +/// +/// Provides a bounded read-only inventory of schemas registered through IPluginSchemaHost. +/// +/// +/// The explorer deliberately reads only the provider registry and its registered schemas. It never treats +/// an arbitrary schema name as trusted input, and it does not expose the Jellyfin public schema. +/// +public sealed class PluginSchemaExplorerService +{ + private const int MaximumPreviewRows = 200; + + /// Lists registered private schemas with table counts and their total on-disk size. + /// Active PostgreSQL connection string. + /// Cancellation token. + /// The registered private schemas. + public async Task> ListSchemasAsync(string connectionString, CancellationToken cancellationToken) + { + const string sql = "SELECT r.plugin_id, r.plugin_name, r.schema_name, r.protocol_version, r.created_at, r.last_seen_at, r.state, count(c.oid)::int, coalesce(pg_size_pretty(sum(pg_total_relation_size(c.oid))), '0 bytes') FROM jellyfin_provider.plugin_schemas r LEFT JOIN pg_namespace n ON n.nspname = r.schema_name LEFT JOIN pg_class c ON c.relnamespace = n.oid AND c.relkind IN ('r', 'p') GROUP BY r.plugin_id, r.plugin_name, r.schema_name, r.protocol_version, r.created_at, r.last_seen_at, r.state ORDER BY r.plugin_name, r.schema_name"; + var result = new List(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + try + { + using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + result.Add(new PluginSchemaInfo(reader.GetGuid(0), reader.GetString(1), reader.GetString(2), reader.GetInt32(3), reader.GetDateTime(4), reader.GetDateTime(5), reader.GetString(6), reader.GetInt32(7), reader.GetString(8))); + } + } + catch (PostgresException ex) when (ex.SqlState == "42P01") + { + return result; + } + + return result; + } + + /// Lists tables that belong to one registered private schema. + /// Active PostgreSQL connection string. + /// Registered private schema name. + /// Cancellation token. + /// The schema's tables. + public async Task> ListTablesAsync(string connectionString, string schemaName, CancellationToken cancellationToken) + { + await EnsureRegisteredAsync(connectionString, schemaName, cancellationToken).ConfigureAwait(false); + const string sql = "SELECT c.relname, c.reltuples::bigint, pg_size_pretty(pg_total_relation_size(c.oid)) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = @schema AND c.relkind IN ('r', 'p') ORDER BY pg_total_relation_size(c.oid) DESC, c.relname"; + var result = new List(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + result.Add(new PluginSchemaTableInfo(reader.GetString(0), reader.GetInt64(1), reader.GetString(2))); + } + + return result; + } + + /// Lists columns of one table inside a registered private schema. + /// Active PostgreSQL connection string. + /// Registered private schema name. + /// Existing table name in that schema. + /// Cancellation token. + /// Column metadata ordered as stored. + public async Task> ListColumnsAsync(string connectionString, string schemaName, string tableName, CancellationToken cancellationToken) + { + await EnsureTableAsync(connectionString, schemaName, tableName, cancellationToken).ConfigureAwait(false); + const string sql = "SELECT column_name, data_type, is_nullable = 'YES', ordinal_position FROM information_schema.columns WHERE table_schema = @schema AND table_name = @table ORDER BY ordinal_position"; + var result = new List(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + command.Parameters.AddWithValue("table", tableName); + using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + result.Add(new PluginSchemaColumnInfo(reader.GetString(0), reader.GetString(1), reader.GetBoolean(2), reader.GetInt32(3))); + } + + return result; + } + + /// Reads a small preview from a verified private plugin table. + /// Active PostgreSQL connection string. + /// Registered private schema name. + /// Existing table name in that schema. + /// Requested number of rows, capped to a safe bound. + /// Cancellation token. + /// Columns and at most rows. + [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "Both identifiers are read from PostgreSQL catalog queries after verifying that the schema is registered and the table belongs to it. Values are not interpolated into this command.")] + public async Task PreviewRowsAsync(string connectionString, string schemaName, string tableName, int limit, CancellationToken cancellationToken) + { + await EnsureTableAsync(connectionString, schemaName, tableName, cancellationToken).ConfigureAwait(false); + var boundedLimit = Math.Clamp(limit, 1, MaximumPreviewRows); + var sql = string.Concat("SELECT * FROM ", QuoteIdentifier(schemaName), ".", QuoteIdentifier(tableName), " LIMIT @limit"); + var columns = new List(); + var rows = new List>(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("limit", boundedLimit + 1); + using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + for (var ordinal = 0; ordinal < reader.FieldCount; ordinal++) + { + columns.Add(reader.GetName(ordinal)); + } + + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false) && rows.Count <= boundedLimit) + { + var row = new string?[reader.FieldCount]; + for (var ordinal = 0; ordinal < reader.FieldCount; ordinal++) + { + row[ordinal] = await reader.IsDBNullAsync(ordinal, cancellationToken).ConfigureAwait(false) ? null : FormatValue(reader.GetValue(ordinal)); + } + + rows.Add(row); + } + + var truncated = rows.Count > boundedLimit; + if (truncated) + { + rows.RemoveAt(rows.Count - 1); + } + + return new PluginSchemaRows(columns, rows, truncated); + } + + private static async Task EnsureRegisteredAsync(string connectionString, string schemaName, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(schemaName); + const string sql = "SELECT EXISTS (SELECT 1 FROM jellyfin_provider.plugin_schemas WHERE schema_name = @schema)"; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + var registered = (bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!; + if (!registered) + { + throw new ArgumentException("El esquema solicitado no está registrado por PG Provider.", nameof(schemaName)); + } + } + + private static async Task EnsureTableAsync(string connectionString, string schemaName, string tableName, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tableName); + await EnsureRegisteredAsync(connectionString, schemaName, cancellationToken).ConfigureAwait(false); + const string sql = "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = @schema AND table_name = @table AND table_type = 'BASE TABLE')"; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + command.Parameters.AddWithValue("table", tableName); + if (!(bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!) + { + throw new ArgumentException("La tabla solicitada no pertenece al esquema registrado.", nameof(tableName)); + } + } + + private static string QuoteIdentifier(string identifier) => string.Concat("\"", identifier.Replace("\"", "\"\"", StringComparison.Ordinal), "\""); + + private static string FormatValue(object value) + => value switch + { + byte[] bytes => string.Create(CultureInfo.InvariantCulture, $"byte[{bytes.Length}]"), + Array array => string.Create(CultureInfo.InvariantCulture, $"[{array.Length} elementos]"), + _ => Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty, + }; +} diff --git a/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs b/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs index f77d563..b8fcb2d 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs @@ -441,7 +441,7 @@ private static void ValidateMigrations(IReadOnlyList migr private static void ValidateDdlSql(string sql) => ValidateSql(sql, "CREATE", "ALTER", "DROP"); - private static void ValidateDataSql(string sql) => ValidateSql(sql, "INSERT", "UPDATE", "DELETE"); + private static void ValidateDataSql(string sql) => ValidateSql(sql, "WITH", "INSERT", "UPDATE", "DELETE"); private static void ValidateQuerySql(string sql) => ValidateSql(sql, "SELECT", "WITH"); @@ -459,7 +459,8 @@ private static void ValidateSql(string sql, params string[] allowedPrefixes) throw new ArgumentException("The SQL operation is not allowed by the private-schema contract.", nameof(sql)); } - if (normalized.Contains(';', StringComparison.Ordinal) || ForbiddenSqlRegex().IsMatch(normalized) || QualifiedIdentifierRegex().IsMatch(normalized)) + var withoutExcludedReferences = ExcludedIdentifierRegex().Replace(normalized, string.Empty); + if (normalized.Contains(';', StringComparison.Ordinal) || ForbiddenSqlRegex().IsMatch(normalized) || QualifiedIdentifierRegex().IsMatch(withoutExcludedReferences)) { throw new ArgumentException("SQL must be one unqualified statement limited to the plugin's private schema.", nameof(sql)); } @@ -481,4 +482,7 @@ private static void ValidateSql(string sql, params string[] allowedPrefixes) [GeneratedRegex("(?:\\\"[A-Za-z_][A-Za-z0-9_]*\\\"|[A-Za-z_][A-Za-z0-9_]*)\\s*\\.", RegexOptions.CultureInvariant)] private static partial Regex QualifiedIdentifierRegex(); + + [GeneratedRegex("\\bexcluded\\s*\\.", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex ExcludedIdentifierRegex(); } diff --git a/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html b/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html index 37107e8..b7fe51f 100644 --- a/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html +++ b/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html @@ -243,6 +243,7 @@

PostgreSQL Databa + @@ -619,6 +620,51 @@

Historial

+ + + @@ -781,6 +827,20 @@

Historial

health_sev_warn: 'Warnings detected', health_sev_error: 'Critical issues detected', tab_query: 'SQL console', + tab_schemas: 'Plugin schemas', + heading_schemas: 'Private plugin schemas', + desc_schemas: 'Read-only inventory of schemas registered through PG Provider. Select a table to inspect its columns and a bounded data preview.', + button_refresh_schemas: 'Refresh', + msg_schemas_loading: 'Loading registered schemas...', + msg_schemas_empty: 'No plugin has registered a private schema yet.', + msg_schemas_error: 'Could not load plugin schemas: ', + msg_schemas_tables: 'Tables: ', + msg_schemas_rows: 'rows', + msg_schemas_preview: 'Preview', + msg_schemas_truncated: 'Preview limited to 200 rows.', + msg_schemas_sort_hint: 'Click a column to sort the visible rows.', + field_plugin_schema: 'Registered schema', + field_plugin_table: 'Table', heading_query: 'Read-only SQL console', desc_query: 'Run read-only statements against PostgreSQL: SELECT, WITH, TABLE, VALUES, EXPLAIN or SHOW. One statement per query, a bounded row count and a 30 second timeout. Everything executed here is written to the plugin log.', field_query_template: 'Template', @@ -978,6 +1038,20 @@

Historial

health_sev_warn: 'Advertencias detectadas', health_sev_error: 'Problemas criticos detectados', tab_query: 'Consola SQL', + tab_schemas: 'Esquemas', + heading_schemas: 'Esquemas privados de plugins', + desc_schemas: 'Inventario de solo lectura de los esquemas registrados mediante PG Provider. Selecciona una tabla para ver columnas y una muestra acotada de datos.', + button_refresh_schemas: 'Actualizar', + msg_schemas_loading: 'Cargando esquemas registrados...', + msg_schemas_empty: 'Ningún plugin ha registrado todavía un esquema privado.', + msg_schemas_error: 'No se pudieron cargar los esquemas de plugins: ', + msg_schemas_tables: 'Tablas: ', + msg_schemas_rows: 'filas', + msg_schemas_preview: 'Muestra', + msg_schemas_truncated: 'La muestra está limitada a 200 filas.', + msg_schemas_sort_hint: 'Haz clic en una columna para ordenar las filas visibles.', + field_plugin_schema: 'Esquema registrado', + field_plugin_table: 'Tabla', heading_query: 'Consola SQL de solo lectura', desc_query: 'Ejecuta sentencias de solo lectura sobre PostgreSQL: SELECT, WITH, TABLE, VALUES, EXPLAIN o SHOW. Una sentencia por consulta, con limite de filas y 30 s de timeout. Todo lo que se ejecuta queda registrado en el log del plugin.', field_query_template: 'Plantilla', @@ -1179,6 +1253,15 @@

Historial

if (tabs.length >= 5) { tabs[4].innerHTML = '📋 ' + t('tab_query', 'SQL console'); } + if (tabs.length >= 6) { + tabs[5].innerHTML = '🗃 ' + t('tab_schemas', 'Plugin schemas'); + } + + setText('schemasHeading', t('heading_schemas', 'Private plugin schemas')); + setText('schemasDescription', t('desc_schemas', 'Read-only inventory of schemas registered through PG Provider.')); + setText('btnRefreshSchemasLabel', t('button_refresh_schemas', 'Refresh')); + setInputLabel('pluginSchemaSelect', t('field_plugin_schema', 'Registered schema')); + setInputLabel('pluginTableSelect', t('field_plugin_table', 'Table')); setText('queryHeading', t('heading_query', 'Read-only SQL console')); setHtml('queryDescription', t('desc_query', 'Run read-only statements against PostgreSQL.')); @@ -1258,6 +1341,7 @@

Historial

btn.classList.add('button-submit'); if (target === 'maintenance') { loadStats(); loadBackupList(); } if (target === 'health') { loadHealthCheck(); } + if (target === 'schemas') { loadPluginSchemas(); } if (target === 'migration') { // Pre-fill export target path with the auto-detected native SQLite path var exportField = document.getElementById('exportTargetPath'); @@ -1530,6 +1614,190 @@

Historial

if ((event.ctrlKey || event.metaKey) && event.key === 'Enter') { event.preventDefault(); runQuery(); } }); + // ── Registered private plugin schemas ─────────────────────────── + // These endpoints are intentionally separate from the general SQL console: the server only + // permits schemas registered by IPluginSchemaHost and only bounded read-only previews. + function schemaValue(item, pascal, camel) { + return item[pascal] !== undefined ? item[pascal] : item[camel]; + } + + var pluginSchemas = []; + var pluginTables = []; + var pluginPreview = null; + var pluginSort = { column: -1, direction: 1 }; + + function loadPluginSchemas() { + var state = document.getElementById('schemasStatus'); + var select = document.getElementById('pluginSchemaSelect'); + var tableSelect = document.getElementById('pluginTableSelect'); + state.style.color = '#90caf9'; + state.textContent = t('msg_schemas_loading', 'Loading registered schemas...'); + select.disabled = true; + tableSelect.disabled = true; + select.innerHTML = ''; + tableSelect.innerHTML = ''; + document.getElementById('pluginSchemaDetails').style.display = 'none'; + apiGet('/PluginSchemas') + .then(function(response) { return response.json(); }) + .then(function(data) { + pluginSchemas = data.Schemas !== undefined ? data.Schemas : (data.schemas || []); + if (!pluginSchemas.length) { + state.style.color = '#fff9c4'; + state.textContent = t('msg_schemas_empty', 'No plugin has registered a private schema yet.'); + return; + } + + state.style.color = '#a5d6a7'; + state.textContent = pluginSchemas.length + ' ' + t('tab_schemas', 'Plugin schemas'); + pluginSchemas.forEach(function(schema, index) { + var option = document.createElement('option'); + var name = schemaValue(schema, 'PluginName', 'pluginName') || '?'; + var schemaName = schemaValue(schema, 'SchemaName', 'schemaName') || ''; + var tables = schemaValue(schema, 'TableCount', 'tableCount') || 0; + var size = schemaValue(schema, 'TotalSize', 'totalSize') || '0 bytes'; + option.value = String(index); + option.textContent = name + ' · ' + schemaName + ' · ' + tables + ' tablas · ' + size; + select.appendChild(option); + }); + select.disabled = false; + select.value = '0'; + loadSelectedPluginSchema(); + }) + .catch(function(error) { + state.style.color = '#ef9a9a'; + state.textContent = t('msg_schemas_error', 'Could not load plugin schemas: ') + error; + }); + } + + function loadSelectedPluginSchema() { + var index = parseInt(document.getElementById('pluginSchemaSelect').value, 10); + var schema = pluginSchemas[index]; + if (!schema) return; + var schemaName = schemaValue(schema, 'SchemaName', 'schemaName'); + var pluginName = schemaValue(schema, 'PluginName', 'pluginName'); + var details = document.getElementById('pluginSchemaDetails'); + var tableSelect = document.getElementById('pluginTableSelect'); + details.style.display = ''; + document.getElementById('selectedSchemaHeading').textContent = pluginName + ' · ' + schemaName; + document.getElementById('selectedSchemaMeta').textContent = (schemaValue(schema, 'TableCount', 'tableCount') || 0) + ' ' + t('msg_schemas_tables', 'Tables: ') + ' · ' + (schemaValue(schema, 'TotalSize', 'totalSize') || '0 bytes'); + document.getElementById('pluginTableDetails').style.display = 'none'; + tableSelect.disabled = true; + tableSelect.innerHTML = ''; + apiGet('/PluginSchemas/' + encodeURIComponent(schemaName) + '/Tables') + .then(function(response) { return response.json(); }) + .then(function(tables) { + pluginTables = tables || []; + pluginTables.forEach(function(table, tableIndex) { + var option = document.createElement('option'); + var tableName = schemaValue(table, 'TableName', 'tableName') || ''; + var rows = schemaValue(table, 'EstimatedRows', 'estimatedRows') || 0; + var size = schemaValue(table, 'TotalSize', 'totalSize') || ''; + option.value = String(tableIndex); + option.textContent = tableName + ' · ' + rows + ' ' + t('msg_schemas_rows', 'rows') + ' · ' + size; + tableSelect.appendChild(option); + }); + tableSelect.disabled = !pluginTables.length; + if (pluginTables.length) { + tableSelect.value = '0'; + loadSelectedPluginTable(); + } + }) + .catch(function(error) { document.getElementById('schemasStatus').textContent = t('msg_schemas_error', 'Could not load plugin schemas: ') + error; }); + } + + function loadSelectedPluginTable() { + var schema = pluginSchemas[parseInt(document.getElementById('pluginSchemaSelect').value, 10)]; + var table = pluginTables[parseInt(document.getElementById('pluginTableSelect').value, 10)]; + if (!schema || !table) return; + var schemaName = schemaValue(schema, 'SchemaName', 'schemaName'); + var tableName = schemaValue(table, 'TableName', 'tableName'); + var encoded = '/PluginSchemas/' + encodeURIComponent(schemaName) + '/Tables/' + encodeURIComponent(tableName); + document.getElementById('pluginTableDetails').style.display = ''; + document.getElementById('selectedTableHeading').textContent = tableName; + document.getElementById('selectedTableMeta').textContent = (schemaValue(table, 'EstimatedRows', 'estimatedRows') || 0) + ' ' + t('msg_schemas_rows', 'rows') + ' · ' + (schemaValue(table, 'TotalSize', 'totalSize') || ''); + document.getElementById('pluginColumns').textContent = t('msg_schemas_loading', 'Loading registered schemas...'); + Promise.all([apiGet(encoded + '/Columns').then(function(response) { return response.json(); }), apiGet(encoded + '/Rows?limit=100').then(function(response) { return response.json(); })]) + .then(function(results) { + var columns = results[0] || []; + var preview = results[1] || {}; + document.getElementById('pluginColumns').textContent = columns.map(function(column) { + var name = schemaValue(column, 'ColumnName', 'columnName'); + var type = schemaValue(column, 'DataType', 'dataType'); + var nullable = schemaValue(column, 'Nullable', 'nullable'); + return name + ' · ' + type + (nullable ? '' : ' NOT NULL'); + }).join(' | '); + document.getElementById('pluginColumnsSummary').textContent = columns.length + ' columnas'; + renderPluginRows(preview); + }) + .catch(function(error) { document.getElementById('pluginColumns').textContent = t('msg_schemas_error', 'Could not load plugin schemas: ') + error; }); + } + + function renderPluginRows(preview) { + var head = document.getElementById('pluginRowsHead'); + var body = document.getElementById('pluginRowsBody'); + var columns = preview.Columns !== undefined ? preview.Columns : (preview.columns || []); + var rows = preview.Rows !== undefined ? preview.Rows : (preview.rows || []); + pluginPreview = { columns: columns, rows: rows, truncated: preview.Truncated || preview.truncated }; + pluginSort = { column: -1, direction: 1 }; + renderSortedPluginRows(); + } + + function renderSortedPluginRows() { + if (!pluginPreview) return; + var head = document.getElementById('pluginRowsHead'); + var body = document.getElementById('pluginRowsBody'); + var columns = pluginPreview.columns; + var rows = pluginPreview.rows.slice(); + if (pluginSort.column >= 0) { + rows.sort(function(left, right) { return comparePluginValues(left[pluginSort.column], right[pluginSort.column]) * pluginSort.direction; }); + } + head.innerHTML = ''; + body.innerHTML = ''; + columns.forEach(function(column, columnIndex) { + var cell = document.createElement('th'); + cell.style.cssText = 'text-align:left; padding:0; position:sticky; top:0; background:#131c33; border-bottom:1px solid rgba(124,146,196,.35); white-space:nowrap;'; + var sort = document.createElement('button'); + sort.type = 'button'; + sort.className = 'button-link'; + sort.style.cssText = 'width:100%; text-align:left; padding:7px 10px; color:#dbe7ff; cursor:pointer; font-weight:600;'; + sort.textContent = column + (pluginSort.column === columnIndex ? (pluginSort.direction === 1 ? ' ▲' : ' ▼') : ' ↕'); + sort.addEventListener('click', function() { sortPluginRows(columnIndex); }); + cell.appendChild(sort); + head.appendChild(cell); + }); + rows.forEach(function(row, index) { + var tr = document.createElement('tr'); + tr.style.background = index % 2 === 0 ? 'rgba(255,255,255,.02)' : 'transparent'; + row.forEach(function(value) { + var cell = document.createElement('td'); + cell.style.cssText = 'padding:5px 10px; border-bottom:1px solid rgba(124,146,196,.12); white-space:nowrap; max-width:520px; overflow:hidden; text-overflow:ellipsis;'; + cell.textContent = value === null || value === undefined ? 'NULL' : value; + tr.appendChild(cell); + }); + body.appendChild(tr); + }); + document.getElementById('pluginRowsHint').textContent = rows.length + ' ' + t('msg_schemas_rows', 'rows') + ' · ' + t('msg_schemas_sort_hint', 'Click a column to sort the visible rows.') + (pluginPreview.truncated ? ' · ' + t('msg_schemas_truncated', 'Preview limited to 200 rows.') : ''); + } + + function sortPluginRows(column) { + if (pluginSort.column === column) pluginSort.direction *= -1; + else pluginSort = { column: column, direction: 1 }; + renderSortedPluginRows(); + } + + function comparePluginValues(left, right) { + if (left === null || left === undefined) return right === null || right === undefined ? 0 : -1; + if (right === null || right === undefined) return 1; + var leftNumber = Number(left); + var rightNumber = Number(right); + if (!isNaN(leftNumber) && !isNaN(rightNumber)) return leftNumber - rightNumber; + return String(left).localeCompare(String(right), undefined, { numeric: true, sensitivity: 'base' }); + } + + document.getElementById('btnRefreshSchemas').addEventListener('click', loadPluginSchemas); + document.getElementById('pluginSchemaSelect').addEventListener('change', loadSelectedPluginSchema); + document.getElementById('pluginTableSelect').addEventListener('change', loadSelectedPluginTable); + function loadStatus() { hideBanner(); apiGet('/Status').then(function(resp) { return resp.json(); }).then(function(data) { From 3f37de37774bf2d2ad6409f0918db11591b48340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20Ch=C3=A1vez=20Bornios?= Date: Sun, 27 Sep 2026 22:17:29 -0700 Subject: [PATCH 04/12] refine: streamline plugin schema explorer --- .../Web/configurationPage.html | 104 ++++++++++-------- 1 file changed, 56 insertions(+), 48 deletions(-) diff --git a/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html b/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html index b7fe51f..6165c5b 100644 --- a/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html +++ b/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html @@ -641,19 +641,7 @@

Esquemas privados de plugins

 /// /// A consumer obtains by asking Jellyfin's service provider for that -/// interface type from this loaded assembly. Consumers that cannot reference this assembly at compile time -/// may discover this marker and the interface by their full names through reflection. +/// interface type from this loaded assembly. In a plugin load-context mismatch, consumers should first +/// locate PG Provider's live plugin assembly through IPluginManager.Plugins, then request the +/// type named by from that assembly. Consumers must not construct an +/// internal provider implementation through reflection or ActivatorUtilities. /// public static class PostgresSchemaHostCapability { diff --git a/Jellyfin.Database.Providers.Postgres/Controllers/Models/PluginSchemaMaintenancePolicyRequest.cs b/Jellyfin.Database.Providers.Postgres/Controllers/Models/PluginSchemaMaintenancePolicyRequest.cs new file mode 100644 index 0000000..7ca1a27 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Controllers/Models/PluginSchemaMaintenancePolicyRequest.cs @@ -0,0 +1,14 @@ +namespace Jellyfin.Database.Providers.Postgres.Controllers.Models; + +/// Administrator-selected scheduled-maintenance policy for a private plugin schema. +public sealed class PluginSchemaMaintenancePolicyRequest +{ + /// Gets or sets a value indicating whether scheduled provider maintenance refreshes planner statistics. + public bool IncludeAnalyze { get; set; } + + /// Gets or sets a value indicating whether scheduled provider maintenance runs VACUUM (ANALYZE). + public bool IncludeVacuum { get; set; } + + /// Gets or sets a value indicating whether scheduled provider maintenance rebuilds indexes concurrently. + public bool IncludeReindex { get; set; } +} diff --git a/Jellyfin.Database.Providers.Postgres/Controllers/PostgresController.cs b/Jellyfin.Database.Providers.Postgres/Controllers/PostgresController.cs index 12bc251..2b241cf 100644 --- a/Jellyfin.Database.Providers.Postgres/Controllers/PostgresController.cs +++ b/Jellyfin.Database.Providers.Postgres/Controllers/PostgresController.cs @@ -19,6 +19,8 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +#pragma warning disable SA1611, SA1615 + namespace Jellyfin.Database.Providers.Postgres.Controllers; /// @@ -40,6 +42,7 @@ public class PostgresController : ControllerBase private readonly InstantSearchService _instantSearch; private readonly ExportToSqliteService _exportService; private readonly PluginSchemaExplorerService _pluginSchemaExplorer; + private readonly PluginSchemaMaintenanceService _pluginSchemaMaintenance; private readonly ILogger _logger; /// @@ -52,6 +55,7 @@ public class PostgresController : ControllerBase /// Instant search service instance. /// Export service instance. /// Read-only explorer for registered private plugin schemas. + /// Scoped maintenance for registered private plugin schemas. /// Logger instance. public PostgresController( IApplicationPaths appPaths, @@ -61,6 +65,7 @@ public PostgresController( InstantSearchService instantSearch, ExportToSqliteService exportService, PluginSchemaExplorerService pluginSchemaExplorer, + PluginSchemaMaintenanceService pluginSchemaMaintenance, ILogger logger) { _appPaths = appPaths; @@ -70,6 +75,7 @@ public PostgresController( _instantSearch = instantSearch; _exportService = exportService; _pluginSchemaExplorer = pluginSchemaExplorer; + _pluginSchemaMaintenance = pluginSchemaMaintenance; _logger = logger; } @@ -1109,6 +1115,59 @@ public async Task> GetPluginSchemaRows(string schemaName, s => await ExecuteSchemaInspectionAsync( connectionString => _pluginSchemaExplorer.PreviewRowsAsync(connectionString, schemaName, tableName, limit, cancellationToken)).ConfigureAwait(false); + /// Gets maintenance signals for a registered private plugin schema. + /// Registered schema to inspect. + /// Cancellation token. + /// Per-table statistics and conservative maintenance recommendations. + [HttpGet("PluginSchemas/{schemaName}/Maintenance")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task> GetPluginSchemaMaintenance(string schemaName, CancellationToken cancellationToken) + => await ExecuteSchemaInspectionAsync( + connectionString => _pluginSchemaMaintenance.GetReportAsync(connectionString, schemaName, cancellationToken)).ConfigureAwait(false); + + /// Refreshes planner statistics for all tables in a registered private schema. + [HttpPost("PluginSchemas/{schemaName}/Maintenance/Analyze")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task> AnalyzePluginSchema(string schemaName, CancellationToken cancellationToken) + => await ExecuteSchemaMaintenanceAsync( + connectionString => _pluginSchemaMaintenance.AnalyzeAsync(connectionString, schemaName, cancellationToken)).ConfigureAwait(false); + + /// Runs VACUUM (ANALYZE) for one table in a registered private schema. + [HttpPost("PluginSchemas/{schemaName}/Tables/{tableName}/Maintenance/VacuumAnalyze")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task> VacuumAnalyzePluginSchemaTable(string schemaName, string tableName, CancellationToken cancellationToken) + => await ExecuteSchemaMaintenanceAsync( + connectionString => _pluginSchemaMaintenance.VacuumAnalyzeTableAsync(connectionString, schemaName, tableName, cancellationToken)).ConfigureAwait(false); + + /// Rebuilds the indexes of one table in a registered private schema concurrently. + [HttpPost("PluginSchemas/{schemaName}/Tables/{tableName}/Maintenance/Reindex")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task> ReindexPluginSchemaTable(string schemaName, string tableName, CancellationToken cancellationToken) + => await ExecuteSchemaMaintenanceAsync( + connectionString => _pluginSchemaMaintenance.ReindexTableAsync(connectionString, schemaName, tableName, cancellationToken)).ConfigureAwait(false); + + /// Stores the administrator-selected scheduled maintenance policy for a private plugin schema. + [HttpPost("PluginSchemas/{schemaName}/Maintenance/Policy")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesResponseType(StatusCodes.Status400BadRequest)] + public async Task> UpdatePluginSchemaMaintenancePolicy( + string schemaName, + [FromBody] PluginSchemaMaintenancePolicyRequest request, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + return await ExecuteSchemaMaintenanceAsync( + connectionString => _pluginSchemaMaintenance.UpdatePolicyAsync( + connectionString, + schemaName, + new PluginSchemaMaintenancePolicy(request.IncludeAnalyze, request.IncludeVacuum, request.IncludeReindex), + cancellationToken)).ConfigureAwait(false); + } + // ───────────────────────────────────────────────────────────────────────── // Read-only SQL console // ───────────────────────────────────────────────────────────────────────── @@ -1217,6 +1276,36 @@ private async Task> ExecuteSchemaInspectionAsync(Func> ExecuteSchemaMaintenanceAsync(Func> operation) + { + ArgumentNullException.ThrowIfNull(operation); + var connectionString = GetActiveConnectionString(); + if (connectionString is null) + { + return NoActiveConnection(); + } + + try + { + return Ok(await operation(connectionString).ConfigureAwait(false)); + } + catch (ArgumentException ex) + { + Logging.PostgresLog.Warn($"[PluginSchemas] Mantenimiento rechazado: {ex.Message}"); + return BadRequest(new { Error = ex.Message }); + } + catch (Npgsql.PostgresException ex) + { + Logging.PostgresLog.Error($"[PluginSchemas] PostgreSQL rechazó el mantenimiento ({ex.SqlState})", ex); + return BadRequest(new { Error = $"({ex.SqlState}) {ex.MessageText}" }); + } + catch (Exception ex) + { + Logging.PostgresLog.Error("[PluginSchemas] Error en mantenimiento de esquema privado", ex); + return BadRequest(new { Error = ex.Message }); + } + } + private ObjectResult NoActiveConnection() => StatusCode( StatusCodes.Status409Conflict, @@ -1392,3 +1481,4 @@ private static string ResolveExportFilePath(string rawPath) return full; } } +#pragma warning restore SA1611, SA1615 diff --git a/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs b/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs index 13efe28..feca51c 100644 --- a/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs +++ b/Jellyfin.Database.Providers.Postgres/PostgresServiceRegistrator.cs @@ -30,6 +30,7 @@ public void RegisterServices(IServiceCollection serviceCollection, IServerApplic serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); + serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); serviceCollection.AddSingleton(); diff --git a/Jellyfin.Database.Providers.Postgres/Services/DateTimeKindNormalizingInterceptor.cs b/Jellyfin.Database.Providers.Postgres/Services/DateTimeKindNormalizingInterceptor.cs index 2bd057d..7f90f26 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/DateTimeKindNormalizingInterceptor.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/DateTimeKindNormalizingInterceptor.cs @@ -33,6 +33,11 @@ namespace Jellyfin.Database.Providers.Postgres.Services; /// public sealed class DateTimeKindNormalizingInterceptor : DbCommandInterceptor { + // Android's java.time cannot represent DateTime.MinValue after a timezone conversion. Npgsql also + // maps this sentinel to PostgreSQL -infinity by default, which can later reach clients as an invalid + // ISO date. Preserve a valid, explicit sentinel instead. + private static readonly DateTime ClientSafeSentinelUtc = DateTime.UnixEpoch; + /// public override InterceptionResult NonQueryExecuting( DbCommand command, @@ -124,10 +129,17 @@ private static void NormalizeParameters(DbCommand command) } private static DateTime NormalizeDateTime(DateTime value) - => value.Kind switch + { + if (value == DateTime.MinValue) + { + return ClientSafeSentinelUtc; + } + + return value.Kind switch { DateTimeKind.Utc => value, DateTimeKind.Local => value.ToUniversalTime(), _ => DateTime.SpecifyKind(value, DateTimeKind.Utc), }; + } } diff --git a/Jellyfin.Database.Providers.Postgres/Services/MaintenanceService.cs b/Jellyfin.Database.Providers.Postgres/Services/MaintenanceService.cs index ec579f7..94c9602 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/MaintenanceService.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/MaintenanceService.cs @@ -13,6 +13,8 @@ using Microsoft.Extensions.Logging; using Npgsql; +#pragma warning disable SA1611, SA1615, SA1503 + namespace Jellyfin.Database.Providers.Postgres.Services; /// @@ -83,6 +85,21 @@ public async Task VacuumAnalyzeAsync(string connectionString, CancellationToken PostgresLog.Info($"[Vacuum] COMPLETADO en {sw.Elapsed.TotalSeconds:F1}s. Las tablas han sido analizadas y el espacio muerto recuperado."); } + /// Runs VACUUM (ANALYZE) one table at a time for a validated provider-owned schema. + [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "The schema is validated against a strict PostgreSQL identifier allow-list and table names come only from pg_class before both are quoted.")] + public async Task VacuumAnalyzeSchemaAsync(string connectionString, string schema, CancellationToken ct = default) + { + var safeSchema = RequireSchemaIdentifier(schema); + var tables = await ListSchemaTablesAsync(connectionString, safeSchema, ct).ConfigureAwait(false); + using var pg = new NpgsqlConnection(connectionString); + await pg.OpenAsync(ct).ConfigureAwait(false); + foreach (var table in tables) + { + using var command = new NpgsqlCommand($"VACUUM (ANALYZE) {QuoteIdentifier(safeSchema)}.{QuoteIdentifier(table)}", pg) { CommandTimeout = 0 }; + await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + } + } + // ── REINDEX ─────────────────────────────────────────────────────────────── /// Runs REINDEX DATABASE CONCURRENTLY. @@ -117,6 +134,17 @@ public async Task ReindexAsync(string connectionString, CancellationToken ct = d PostgresLog.Info($"[Reindex] COMPLETADO en {sw.Elapsed.TotalSeconds:F1}s. Todos los índices de '{dbName}' han sido reconstruidos."); } + /// Rebuilds all indexes in a provider-owned schema concurrently. + [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "The schema is validated against a strict PostgreSQL identifier allow-list before it is quoted.")] + public async Task ReindexSchemaAsync(string connectionString, string schema, CancellationToken ct = default) + { + var safeSchema = RequireSchemaIdentifier(schema); + using var pg = new NpgsqlConnection(connectionString); + await pg.OpenAsync(ct).ConfigureAwait(false); + using var command = new NpgsqlCommand($"REINDEX SCHEMA CONCURRENTLY {QuoteIdentifier(safeSchema)}", pg) { CommandTimeout = 0 }; + await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + } + // ── Table statistics ────────────────────────────────────────────────────── /// Returns size and row-count statistics for all user tables in the given schema. @@ -233,6 +261,29 @@ AND c.relname LIKE '%_gin_trgm' internal static string QuoteIdentifier(string input) => $"\"{input.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; + private static async Task> ListSchemaTablesAsync(string connectionString, string schema, CancellationToken ct) + { + const string sql = "SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = @schema AND c.relkind IN ('r', 'p') ORDER BY c.relname"; + var tables = new List(); + using var pg = new NpgsqlConnection(connectionString); + await pg.OpenAsync(ct).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, pg); + command.Parameters.AddWithValue("schema", schema); + using var reader = await command.ExecuteReaderAsync(ct).ConfigureAwait(false); + while (await reader.ReadAsync(ct).ConfigureAwait(false)) tables.Add(reader.GetString(0)); + return tables; + } + + private static string RequireSchemaIdentifier(string value) + { + if (string.IsNullOrWhiteSpace(value) || !System.Text.RegularExpressions.Regex.IsMatch(value, "^[a-z][a-z0-9_]{0,62}$", System.Text.RegularExpressions.RegexOptions.CultureInvariant)) + { + throw new ArgumentException("Invalid PostgreSQL schema identifier.", nameof(value)); + } + + return value; + } + internal static string ValidateBackupFilePath(string path) { if (!Path.IsPathFullyQualified(path)) @@ -434,3 +485,4 @@ private static NpgsqlCommand CreateReindexCommand(NpgsqlConnection pg, string db return new NpgsqlCommand(sql, pg) { CommandTimeout = 0 }; } } +#pragma warning restore SA1611, SA1615, SA1503 diff --git a/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaInfo.cs b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaInfo.cs index 3bb6791..2480873 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaInfo.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaInfo.cs @@ -12,4 +12,7 @@ public sealed record PluginSchemaInfo( DateTime LastSeenAt, string State, int TableCount, - string TotalSize); + string TotalSize, + bool IncludeAnalyze, + bool IncludeVacuum, + bool IncludeReindex); diff --git a/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaMaintenancePolicy.cs b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaMaintenancePolicy.cs new file mode 100644 index 0000000..c99ab48 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaMaintenancePolicy.cs @@ -0,0 +1,7 @@ +namespace Jellyfin.Database.Providers.Postgres.Services.Models; + +/// Administrator-controlled schedule policy for one registered private plugin schema. +public sealed record PluginSchemaMaintenancePolicy( + bool IncludeAnalyze, + bool IncludeVacuum, + bool IncludeReindex); diff --git a/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaMaintenanceReport.cs b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaMaintenanceReport.cs new file mode 100644 index 0000000..fcb20e5 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/Models/PluginSchemaMaintenanceReport.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; + +namespace Jellyfin.Database.Providers.Postgres.Services.Models; + +#pragma warning disable SA1402 + +/// Bounded health and maintenance information for one registered private plugin schema. +public sealed record PluginSchemaMaintenanceReport( + string SchemaName, + IReadOnlyList Tables, + int TablesNeedingAnalyze, + int TablesNeedingVacuum, + bool IncludeAnalyze, + bool IncludeVacuum, + bool IncludeReindex, + DateTime CollectedAt); + +/// Operational statistics for one table that belongs to a registered private plugin schema. +public sealed record PluginSchemaMaintenanceTable( + string TableName, + long EstimatedRows, + string TotalSize, + string IndexSize, + long DeadTuples, + long ModifiedSinceAnalyze, + DateTime? LastAnalyze, + DateTime? LastAutoAnalyze, + DateTime? LastVacuum, + DateTime? LastAutoVacuum, + int IndexCount, + bool NeedsAnalyze, + bool NeedsVacuum); + +/// Result of an explicitly requested private-schema maintenance operation. +public sealed record PluginSchemaMaintenanceResult(string Operation, string SchemaName, string? TableName, int TablesAffected, long DurationMs); +#pragma warning restore SA1402 diff --git a/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaExplorerService.cs b/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaExplorerService.cs index a75d5e1..3f91cd6 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaExplorerService.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaExplorerService.cs @@ -9,6 +9,8 @@ using Jellyfin.Database.Providers.Postgres.Services.Models; using Npgsql; +#pragma warning disable SA1513 + namespace Jellyfin.Database.Providers.Postgres.Services; /// @@ -28,17 +30,25 @@ public sealed class PluginSchemaExplorerService /// The registered private schemas. public async Task> ListSchemasAsync(string connectionString, CancellationToken cancellationToken) { - const string sql = "SELECT r.plugin_id, r.plugin_name, r.schema_name, r.protocol_version, r.created_at, r.last_seen_at, r.state, count(c.oid)::int, coalesce(pg_size_pretty(sum(pg_total_relation_size(c.oid))), '0 bytes') FROM jellyfin_provider.plugin_schemas r LEFT JOIN pg_namespace n ON n.nspname = r.schema_name LEFT JOIN pg_class c ON c.relnamespace = n.oid AND c.relkind IN ('r', 'p') GROUP BY r.plugin_id, r.plugin_name, r.schema_name, r.protocol_version, r.created_at, r.last_seen_at, r.state ORDER BY r.plugin_name, r.schema_name"; + const string sql = "SELECT r.plugin_id, r.plugin_name, r.schema_name, r.protocol_version, r.created_at, r.last_seen_at, r.state, count(c.oid)::int, coalesce(pg_size_pretty(sum(pg_total_relation_size(c.oid))), '0 bytes'), r.include_analyze, r.include_vacuum, r.include_reindex FROM jellyfin_provider.plugin_schemas r LEFT JOIN pg_namespace n ON n.nspname = r.schema_name LEFT JOIN pg_class c ON c.relnamespace = n.oid AND c.relkind IN ('r', 'p') GROUP BY r.plugin_id, r.plugin_name, r.schema_name, r.protocol_version, r.created_at, r.last_seen_at, r.state, r.include_analyze, r.include_vacuum, r.include_reindex ORDER BY r.plugin_name, r.schema_name"; var result = new List(); using var connection = new NpgsqlConnection(connectionString); await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + try + { + await EnsurePolicyColumnsAsync(connection, cancellationToken).ConfigureAwait(false); + } + catch (PostgresException ex) when (ex.SqlState == "42P01") + { + return result; + } using var command = new NpgsqlCommand(sql, connection); try { using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { - result.Add(new PluginSchemaInfo(reader.GetGuid(0), reader.GetString(1), reader.GetString(2), reader.GetInt32(3), reader.GetDateTime(4), reader.GetDateTime(5), reader.GetString(6), reader.GetInt32(7), reader.GetString(8))); + result.Add(new PluginSchemaInfo(reader.GetGuid(0), reader.GetString(1), reader.GetString(2), reader.GetInt32(3), reader.GetDateTime(4), reader.GetDateTime(5), reader.GetString(6), reader.GetInt32(7), reader.GetString(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11))); } } catch (PostgresException ex) when (ex.SqlState == "42P01") @@ -49,6 +59,13 @@ public async Task> ListSchemasAsync(string conne return result; } + private static async Task EnsurePolicyColumnsAsync(NpgsqlConnection connection, CancellationToken cancellationToken) + { + const string sql = "ALTER TABLE jellyfin_provider.plugin_schemas ADD COLUMN IF NOT EXISTS include_analyze boolean NOT NULL DEFAULT true; ALTER TABLE jellyfin_provider.plugin_schemas ADD COLUMN IF NOT EXISTS include_vacuum boolean NOT NULL DEFAULT false; ALTER TABLE jellyfin_provider.plugin_schemas ADD COLUMN IF NOT EXISTS include_reindex boolean NOT NULL DEFAULT false;"; + using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + /// Lists tables that belong to one registered private schema. /// Active PostgreSQL connection string. /// Registered private schema name. @@ -183,3 +200,4 @@ private static string FormatValue(object value) _ => Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty, }; } +#pragma warning restore SA1513 diff --git a/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaMaintenanceService.cs b/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaMaintenanceService.cs new file mode 100644 index 0000000..17c3924 --- /dev/null +++ b/Jellyfin.Database.Providers.Postgres/Services/PluginSchemaMaintenanceService.cs @@ -0,0 +1,319 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Database.Providers.Postgres.Logging; +using Jellyfin.Database.Providers.Postgres.Services.Models; +using Npgsql; + +#pragma warning disable SA1611, SA1615 + +namespace Jellyfin.Database.Providers.Postgres.Services; + +/// +/// Performs explicitly requested maintenance on schemas registered through IPluginSchemaHost. +/// +/// +/// This service is intentionally separate from database-wide maintenance. Every public operation verifies +/// the schema in the provider registry, and table operations verify the table in that schema before an +/// identifier is quoted into SQL. It never accepts public or arbitrary PostgreSQL schemas. +/// +public sealed class PluginSchemaMaintenanceService +{ + private const long AnalyzeMinimumChanges = 1_000; + private const long VacuumMinimumDeadTuples = 1_000; + + /// Gets operational statistics for one registered private schema. + public async Task GetReportAsync(string connectionString, string schemaName, CancellationToken cancellationToken) + { + await EnsureRegisteredAsync(connectionString, schemaName, cancellationToken).ConfigureAwait(false); + var policy = await ReadPolicyAsync(connectionString, schemaName, cancellationToken).ConfigureAwait(false); + const string sql = @" +SELECT c.relname, + GREATEST(c.reltuples::bigint, 0), + pg_size_pretty(pg_total_relation_size(c.oid)), + pg_size_pretty(pg_total_relation_size(c.oid) - pg_relation_size(c.oid)), + COALESCE(s.n_dead_tup, 0)::bigint, + COALESCE(s.n_mod_since_analyze, 0)::bigint, + s.last_analyze, + s.last_autoanalyze, + s.last_vacuum, + s.last_autovacuum, + COALESCE(i.index_count, 0)::int +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +LEFT JOIN pg_stat_all_tables s ON s.relid = c.oid +LEFT JOIN LATERAL ( + SELECT count(*)::int AS index_count + FROM pg_index ix + WHERE ix.indrelid = c.oid +) i ON true +WHERE n.nspname = @schema + AND c.relkind IN ('r', 'p') +ORDER BY pg_total_relation_size(c.oid) DESC, c.relname"; + + var tables = new List(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + var estimatedRows = reader.GetInt64(1); + var deadTuples = reader.GetInt64(4); + var modifiedSinceAnalyze = reader.GetInt64(5); + var needsAnalyze = modifiedSinceAnalyze >= Math.Max(AnalyzeMinimumChanges, estimatedRows / 10); + var needsVacuum = deadTuples >= Math.Max(VacuumMinimumDeadTuples, estimatedRows / 5); + tables.Add(new PluginSchemaMaintenanceTable( + reader.GetString(0), + estimatedRows, + reader.GetString(2), + reader.GetString(3), + deadTuples, + modifiedSinceAnalyze, + ReadNullableDateTime(reader, 6), + ReadNullableDateTime(reader, 7), + ReadNullableDateTime(reader, 8), + ReadNullableDateTime(reader, 9), + reader.GetInt32(10), + needsAnalyze, + needsVacuum)); + } + + return new PluginSchemaMaintenanceReport( + schemaName, + tables, + tables.Count(static table => table.NeedsAnalyze), + tables.Count(static table => table.NeedsVacuum), + policy.IncludeAnalyze, + policy.IncludeVacuum, + policy.IncludeReindex, + DateTime.UtcNow); + } + + /// Updates the administrator-controlled scheduled-maintenance policy for one registered schema. + public async Task UpdatePolicyAsync(string connectionString, string schemaName, PluginSchemaMaintenancePolicy policy, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(policy); + await EnsureRegisteredAsync(connectionString, schemaName, cancellationToken).ConfigureAwait(false); + const string sql = "UPDATE jellyfin_provider.plugin_schemas SET include_analyze = @analyze, include_vacuum = @vacuum, include_reindex = @reindex WHERE schema_name = @schema"; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("analyze", policy.IncludeAnalyze); + command.Parameters.AddWithValue("vacuum", policy.IncludeVacuum); + command.Parameters.AddWithValue("reindex", policy.IncludeReindex); + command.Parameters.AddWithValue("schema", schemaName); + if (await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false) != 1) + { + throw new ArgumentException("El esquema solicitado no está registrado por PG Provider.", nameof(schemaName)); + } + + PostgresLog.Info($"[PluginSchemas] Política actualizada: esquema={schemaName}, analyze={policy.IncludeAnalyze}, vacuum={policy.IncludeVacuum}, reindex={policy.IncludeReindex}."); + return policy; + } + + /// Lists registered schemas opted into a scheduled private-schema operation. + public async Task> ListScheduledSchemasAsync(string connectionString, string policyColumn, CancellationToken cancellationToken) + { + var sql = policyColumn switch + { + "analyze" => "SELECT schema_name FROM jellyfin_provider.plugin_schemas WHERE include_analyze = true ORDER BY schema_name", + "vacuum" => "SELECT schema_name FROM jellyfin_provider.plugin_schemas WHERE include_vacuum = true ORDER BY schema_name", + "reindex" => "SELECT schema_name FROM jellyfin_provider.plugin_schemas WHERE include_reindex = true ORDER BY schema_name", + _ => throw new ArgumentException("Política de mantenimiento no reconocida.", nameof(policyColumn)), + }; + var schemas = new List(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + try + { + using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + schemas.Add(reader.GetString(0)); + } + } + catch (PostgresException ex) when (ex.SqlState == "42P01") + { + return schemas; + } + + return schemas; + } + + /// Runs ANALYZE for every table in one registered private schema. + public Task AnalyzeAsync(string connectionString, string schemaName, CancellationToken cancellationToken) + => RunSchemaOperationAsync(connectionString, schemaName, "ANALYZE", "ANALYZE", cancellationToken); + + /// Runs VACUUM (ANALYZE) for one verified private-plugin table. + public Task VacuumAnalyzeTableAsync(string connectionString, string schemaName, string tableName, CancellationToken cancellationToken) + => RunTableOperationAsync(connectionString, schemaName, tableName, "VACUUM (ANALYZE)", "VACUUM (ANALYZE)", cancellationToken); + + /// Rebuilds the indexes for one verified private-plugin table without blocking normal writes. + public Task ReindexTableAsync(string connectionString, string schemaName, string tableName, CancellationToken cancellationToken) + => RunTableOperationAsync(connectionString, schemaName, tableName, "REINDEX CONCURRENTLY", "REINDEX TABLE CONCURRENTLY", cancellationToken); + + private static async Task RunSchemaOperationAsync( + string connectionString, + string schemaName, + string operation, + string statement, + CancellationToken cancellationToken) + { + await EnsureRegisteredAsync(connectionString, schemaName, cancellationToken).ConfigureAwait(false); + var tableNames = await ListTablesAsync(connectionString, schemaName, cancellationToken).ConfigureAwait(false); + var stopwatch = Stopwatch.StartNew(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await AcquireSchemaLockAsync(connection, schemaName, cancellationToken).ConfigureAwait(false); + try + { + foreach (var tableName in tableNames) + { + await ExecuteMaintenanceAsync(connection, statement, schemaName, tableName, cancellationToken).ConfigureAwait(false); + } + } + finally + { + await ReleaseSchemaLockAsync(connection, schemaName).ConfigureAwait(false); + } + + stopwatch.Stop(); + PostgresLog.Info($"[PluginSchemas] {operation} completado: esquema={schemaName}, tablas={tableNames.Count}, duracion={stopwatch.Elapsed.TotalSeconds.ToString("F1", CultureInfo.InvariantCulture)}s."); + return new PluginSchemaMaintenanceResult(operation, schemaName, null, tableNames.Count, stopwatch.ElapsedMilliseconds); + } + + private static async Task RunTableOperationAsync( + string connectionString, + string schemaName, + string tableName, + string operation, + string statement, + CancellationToken cancellationToken) + { + await EnsureTableAsync(connectionString, schemaName, tableName, cancellationToken).ConfigureAwait(false); + var stopwatch = Stopwatch.StartNew(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + await AcquireSchemaLockAsync(connection, schemaName, cancellationToken).ConfigureAwait(false); + try + { + await ExecuteMaintenanceAsync(connection, statement, schemaName, tableName, cancellationToken).ConfigureAwait(false); + } + finally + { + await ReleaseSchemaLockAsync(connection, schemaName).ConfigureAwait(false); + } + + stopwatch.Stop(); + PostgresLog.Info($"[PluginSchemas] {operation} completado: esquema={schemaName}, tabla={tableName}, duracion={stopwatch.Elapsed.TotalSeconds.ToString("F1", CultureInfo.InvariantCulture)}s."); + return new PluginSchemaMaintenanceResult(operation, schemaName, tableName, 1, stopwatch.ElapsedMilliseconds); + } + + [SuppressMessage("Security", "CA2100:Review SQL queries for security vulnerabilities", Justification = "The schema and table were verified against the provider registry and PostgreSQL catalog immediately before execution, then safely quoted.")] + private static async Task ExecuteMaintenanceAsync(NpgsqlConnection connection, string statement, string schemaName, string tableName, CancellationToken cancellationToken) + { + var target = string.Concat(MaintenanceService.QuoteIdentifier(schemaName), ".", MaintenanceService.QuoteIdentifier(tableName)); + using var command = new NpgsqlCommand(string.Concat(statement, " ", target), connection) { CommandTimeout = 0 }; + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + private static async Task> ListTablesAsync(string connectionString, string schemaName, CancellationToken cancellationToken) + { + const string sql = "SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = @schema AND c.relkind IN ('r', 'p') ORDER BY c.relname"; + var names = new List(); + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + names.Add(reader.GetString(0)); + } + + return names; + } + + private static async Task EnsureRegisteredAsync(string connectionString, string schemaName, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(schemaName); + const string sql = "SELECT EXISTS (SELECT 1 FROM jellyfin_provider.plugin_schemas WHERE schema_name = @schema)"; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + if (!(bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!) + { + throw new ArgumentException("El esquema solicitado no está registrado por PG Provider.", nameof(schemaName)); + } + } + + private static async Task ReadPolicyAsync(string connectionString, string schemaName, CancellationToken cancellationToken) + { + const string sql = "SELECT include_analyze, include_vacuum, include_reindex FROM jellyfin_provider.plugin_schemas WHERE schema_name = @schema"; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new ArgumentException("El esquema solicitado no está registrado por PG Provider.", nameof(schemaName)); + } + + return new PluginSchemaMaintenancePolicy(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2)); + } + + private static async Task EnsureTableAsync(string connectionString, string schemaName, string tableName, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tableName); + await EnsureRegisteredAsync(connectionString, schemaName, cancellationToken).ConfigureAwait(false); + const string sql = "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = @schema AND table_name = @table AND table_type = 'BASE TABLE')"; + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + command.Parameters.AddWithValue("table", tableName); + if (!(bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!) + { + throw new ArgumentException("La tabla solicitada no pertenece al esquema registrado.", nameof(tableName)); + } + } + + private static async Task AcquireSchemaLockAsync(NpgsqlConnection connection, string schemaName, CancellationToken cancellationToken) + { + const string sql = "SELECT pg_try_advisory_lock(hashtextextended(@schema, 0))"; + using var command = new NpgsqlCommand(sql, connection); + command.Parameters.AddWithValue("schema", schemaName); + if (!(bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!) + { + throw new InvalidOperationException("Ya hay una operación de mantenimiento en curso para este esquema."); + } + } + + private static async Task ReleaseSchemaLockAsync(NpgsqlConnection connection, string schemaName) + { + try + { + using var command = new NpgsqlCommand("SELECT pg_advisory_unlock(hashtextextended(@schema, 0))", connection); + command.Parameters.AddWithValue("schema", schemaName); + await command.ExecuteNonQueryAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + PostgresLog.Warn($"[PluginSchemas] No se pudo liberar el bloqueo de mantenimiento para {schemaName}: {ex.Message}"); + } + } + + private static DateTime? ReadNullableDateTime(NpgsqlDataReader reader, int ordinal) + => reader.IsDBNull(ordinal) ? null : reader.GetDateTime(ordinal); +} +#pragma warning restore SA1611, SA1615 diff --git a/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs b/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs index b8fcb2d..dfd6439 100644 --- a/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs +++ b/Jellyfin.Database.Providers.Postgres/Services/PostgresPluginSchemaHost.cs @@ -265,6 +265,9 @@ private static async Task EnsureRegistryAsync(NpgsqlConnection connection, Npgsq const string sql = "CREATE TABLE IF NOT EXISTS jellyfin_provider.plugin_schemas (plugin_id uuid PRIMARY KEY, plugin_name text NOT NULL, schema_name text NOT NULL UNIQUE, protocol_version integer NOT NULL, created_at timestamp with time zone NOT NULL DEFAULT now(), last_seen_at timestamp with time zone NOT NULL DEFAULT now(), state text NOT NULL DEFAULT 'active')"; await ExecuteInTransactionAsync(connection, transaction, sql, null, cancellationToken).ConfigureAwait(false); + await ExecuteInTransactionAsync(connection, transaction, "ALTER TABLE jellyfin_provider.plugin_schemas ADD COLUMN IF NOT EXISTS include_analyze boolean NOT NULL DEFAULT true", null, cancellationToken).ConfigureAwait(false); + await ExecuteInTransactionAsync(connection, transaction, "ALTER TABLE jellyfin_provider.plugin_schemas ADD COLUMN IF NOT EXISTS include_vacuum boolean NOT NULL DEFAULT false", null, cancellationToken).ConfigureAwait(false); + await ExecuteInTransactionAsync(connection, transaction, "ALTER TABLE jellyfin_provider.plugin_schemas ADD COLUMN IF NOT EXISTS include_reindex boolean NOT NULL DEFAULT false", null, cancellationToken).ConfigureAwait(false); } private static async Task RegisterSchemaAsync(NpgsqlConnection connection, NpgsqlTransaction transaction, PluginSchemaRequest request, bool existed, CancellationToken cancellationToken) diff --git a/Jellyfin.Database.Providers.Postgres/Tasks/ReindexTask.cs b/Jellyfin.Database.Providers.Postgres/Tasks/ReindexTask.cs index 68fb122..d831d3c 100644 --- a/Jellyfin.Database.Providers.Postgres/Tasks/ReindexTask.cs +++ b/Jellyfin.Database.Providers.Postgres/Tasks/ReindexTask.cs @@ -5,31 +5,38 @@ using Jellyfin.Database.Providers.Postgres.Services; using MediaBrowser.Model.Tasks; +#pragma warning disable SA1611, SA1513 + namespace Jellyfin.Database.Providers.Postgres.Tasks; /// -/// Jellyfin scheduled task that runs REINDEX DATABASE on the PostgreSQL database. +/// Jellyfin scheduled task that rebuilds indexes for Jellyfin and opted-in private plugin schemas. /// Appears in Jellyfin's Scheduled Tasks page for automatic scheduling. /// public class ReindexTask : IScheduledTask { private readonly MaintenanceService _maintenance; + private readonly PluginSchemaMaintenanceService _pluginMaintenance; /// /// Initializes a new instance of the class. /// /// The maintenance service. - public ReindexTask(MaintenanceService maintenance) - => _maintenance = maintenance; + /// The isolated private-schema maintenance service. + public ReindexTask(MaintenanceService maintenance, PluginSchemaMaintenanceService pluginMaintenance) + { + _maintenance = maintenance; + _pluginMaintenance = pluginMaintenance; + } /// - public string Name => "PostgreSQL REINDEX DATABASE"; + public string Name => "PostgreSQL REINDEX (managed schemas)"; /// public string Key => "PostgresReindex"; /// - public string Description => "Runs REINDEX DATABASE on the Jellyfin PostgreSQL database to rebuild all indexes and fix index bloat."; + public string Description => "Rebuilds indexes in Jellyfin public tables and only private plugin schemas opted into scheduled REINDEX."; /// public string Category => "PostgreSQL Maintenance"; @@ -55,7 +62,17 @@ public async Task ExecuteAsync(IProgress progress, CancellationToken can } progress.Report(10); - await _maintenance.ReindexAsync(connStr, cancellationToken).ConfigureAwait(false); + await _maintenance.ReindexSchemaAsync(connStr, "public", cancellationToken).ConfigureAwait(false); + var schemas = await _pluginMaintenance.ListScheduledSchemasAsync(connStr, "reindex", cancellationToken).ConfigureAwait(false); + foreach (var schema in schemas) + { + var tables = await _pluginMaintenance.GetReportAsync(connStr, schema, cancellationToken).ConfigureAwait(false); + foreach (var table in tables.Tables) + { + await _pluginMaintenance.ReindexTableAsync(connStr, schema, table.TableName, cancellationToken).ConfigureAwait(false); + } + } progress.Report(100); } } +#pragma warning restore SA1611, SA1513 diff --git a/Jellyfin.Database.Providers.Postgres/Tasks/VacuumAnalyzeTask.cs b/Jellyfin.Database.Providers.Postgres/Tasks/VacuumAnalyzeTask.cs index 5da69aa..ceb1efb 100644 --- a/Jellyfin.Database.Providers.Postgres/Tasks/VacuumAnalyzeTask.cs +++ b/Jellyfin.Database.Providers.Postgres/Tasks/VacuumAnalyzeTask.cs @@ -1,35 +1,43 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Jellyfin.Database.Providers.Postgres.Services; using MediaBrowser.Model.Tasks; +#pragma warning disable SA1611, SA1513 + namespace Jellyfin.Database.Providers.Postgres.Tasks; /// -/// Jellyfin scheduled task that runs VACUUM ANALYZE on the PostgreSQL database. +/// Jellyfin scheduled task that maintains Jellyfin and opted-in private plugin schemas. /// Appears in Jellyfin's Scheduled Tasks page for automatic scheduling. /// public class VacuumAnalyzeTask : IScheduledTask { private readonly MaintenanceService _maintenance; + private readonly PluginSchemaMaintenanceService _pluginMaintenance; /// /// Initializes a new instance of the class. /// /// The maintenance service. - public VacuumAnalyzeTask(MaintenanceService maintenance) - => _maintenance = maintenance; + /// The isolated private-schema maintenance service. + public VacuumAnalyzeTask(MaintenanceService maintenance, PluginSchemaMaintenanceService pluginMaintenance) + { + _maintenance = maintenance; + _pluginMaintenance = pluginMaintenance; + } /// - public string Name => "PostgreSQL VACUUM ANALYZE"; + public string Name => "PostgreSQL VACUUM ANALYZE (managed schemas)"; /// public string Key => "PostgresVacuumAnalyze"; /// - public string Description => "Runs VACUUM ANALYZE on the Jellyfin PostgreSQL database to reclaim storage and update query-planner statistics."; + public string Description => "Maintains Jellyfin public tables and only private plugin schemas opted into scheduled ANALYZE or VACUUM."; /// public string Category => "PostgreSQL Maintenance"; @@ -55,7 +63,22 @@ public async Task ExecuteAsync(IProgress progress, CancellationToken can } progress.Report(10); - await _maintenance.VacuumAnalyzeAsync(connStr, cancellationToken).ConfigureAwait(false); + await _maintenance.VacuumAnalyzeSchemaAsync(connStr, "public", cancellationToken).ConfigureAwait(false); + var analyzeSchemas = await _pluginMaintenance.ListScheduledSchemasAsync(connStr, "analyze", cancellationToken).ConfigureAwait(false); + var vacuumSchemas = await _pluginMaintenance.ListScheduledSchemasAsync(connStr, "vacuum", cancellationToken).ConfigureAwait(false); + foreach (var schema in analyzeSchemas.Except(vacuumSchemas, StringComparer.Ordinal)) + { + await _pluginMaintenance.AnalyzeAsync(connStr, schema, cancellationToken).ConfigureAwait(false); + } + foreach (var schema in vacuumSchemas) + { + var tables = await _pluginMaintenance.GetReportAsync(connStr, schema, cancellationToken).ConfigureAwait(false); + foreach (var table in tables.Tables) + { + await _pluginMaintenance.VacuumAnalyzeTableAsync(connStr, schema, table.TableName, cancellationToken).ConfigureAwait(false); + } + } progress.Report(100); } } +#pragma warning restore SA1611, SA1513 diff --git a/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html b/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html index 92dd5d0..fe45036 100644 --- a/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html +++ b/Jellyfin.Database.Providers.Postgres/Web/configurationPage.html @@ -243,7 +243,7 @@

PostgreSQL Databa - + @@ -620,25 +620,49 @@

Historial

- +