Merge pull request #24 from JasperFx/weasel-schema-migration

Migrate all DDL to Weasel SchemaMigration

Author: jeremydmiller

src/Polecat.Tests/Storage/document_foreign_key_tests.cs

@@ -216,7 +216,8 @@ await StoreOptions(opts =>
         {
             ConnectionString = theStore.Options.ConnectionString,
             AutoCreateSchemaObjects = JasperFx.AutoCreate.All,
-            DatabaseSchemaName = "fk_idempotent"
+            DatabaseSchemaName = "fk_idempotent",
+            UseNativeJsonType = ConnectionSource.SupportsNativeJson
         };
         opts2.Schema.For<FkUser>();
         opts2.Schema.For<FkIssue>().ForeignKey<FkUser>(x => x.AssigneeId);

src/Polecat.Tests/Storage/document_index_tests.cs

@@ -285,7 +285,8 @@ await StoreOptions(opts =>
         {
             ConnectionString = theStore.Options.ConnectionString,
             AutoCreateSchemaObjects = JasperFx.AutoCreate.All,
-            DatabaseSchemaName = "idx_idempotent"
+            DatabaseSchemaName = "idx_idempotent",
+            UseNativeJsonType = ConnectionSource.SupportsNativeJson
         };
         opts2.Schema.For<IndexedProduct>().Index(x => x.Sku);
         using var store2 = new DocumentStore(opts2);

src/Polecat/AdvancedOperations.cs

@@ -283,35 +283,26 @@ await _resilience.ExecuteAsync(static async (state, ct) =>
     }
 
     /// <summary>
-    ///     Generate the full DDL script for all Polecat schema objects (event store tables + any registered document tables).
+    ///     Generate the full DDL script for all Polecat schema objects (event store tables, document tables, HiLo).
     /// </summary>
     public string ToDatabaseScript()
     {
         var sb = new StringBuilder();
         var writer = new StringWriter(sb);
+        var migrator = new Weasel.SqlServer.SqlServerMigrator();
 
-        // Event store tables via Weasel
+        // All schema objects (event store + documents + hilo) via Weasel feature schemas
         foreach (var featureSchema in _store.Database.BuildFeatureSchemas())
         {
             foreach (var schemaObject in featureSchema.Objects)
             {
-                schemaObject.WriteCreateStatement(new Weasel.SqlServer.SqlServerMigrator(), writer);
+                schemaObject.WriteCreateStatement(migrator, writer);
                 writer.WriteLine();
                 writer.WriteLine("GO");
                 writer.WriteLine();
             }
         }
 
-        // Document tables for already-registered providers
-        foreach (var provider in _store.Options.Providers.AllProviders)
-        {
-            var table = new DocumentTable(provider.Mapping);
-            table.WriteCreateStatement(new Weasel.SqlServer.SqlServerMigrator(), writer);
-            writer.WriteLine();
-            writer.WriteLine("GO");
-            writer.WriteLine();
-        }
-
         return sb.ToString();
     }
 

src/Polecat/Internal/DocumentTableEnsurer.cs

@@ -1,13 +1,16 @@
 using System.Collections.Concurrent;
+using JasperFx;
 using Microsoft.Data.SqlClient;
-using Polecat.Metadata;
+using Polecat.Schema.Identity.Sequences;
 using Polecat.Storage;
+using Weasel.Core;
+using Weasel.SqlServer;
 
 namespace Polecat.Internal;
 
 /// <summary>
-///     Ensures document tables exist on demand. Uses Weasel to create tables
-///     if they don't exist, and tracks which types have been ensured.
+///     Ensures document tables exist on demand. Uses Weasel SchemaMigration to create
+///     or update tables, and tracks which types have been ensured.
 /// </summary>
 internal class DocumentTableEnsurer
 {
@@ -49,35 +52,29 @@ public async Task EnsureTableAsync(DocumentProvider provider, CancellationToken
                 return;
             }
 
-            var table = new DocumentTable(provider.Mapping);
-
             await using var conn = _connectionFactory.Create();
             await conn.OpenAsync(token);
 
+            var migrator = new SqlServerMigrator();
+
             // Ensure pc_hilo table for numeric ID types
             if (provider.Mapping.IsNumericId && !_hiloTableEnsured)
             {
-                var hiloDdl = BuildHiloTableDdl(provider.Mapping.DatabaseSchemaName);
-                await using var hiloCmd = conn.CreateCommand();
-                hiloCmd.CommandText = hiloDdl;
-                await hiloCmd.ExecuteNonQueryAsync(token);
+                var hiloTable = new HiloTable(provider.Mapping.DatabaseSchemaName);
+                var hiloMigration = await SchemaMigration.DetermineAsync(conn, token, hiloTable);
+                await migrator.ApplyAllAsync(conn, hiloMigration, AutoCreate.CreateOrUpdate, ct: token);
                 _hiloTableEnsured = true;
             }
 
-            // Use raw DDL with IF NOT EXISTS for safety
-            var ddl = BuildCreateTableDdl(provider.Mapping);
-            await using var cmd = conn.CreateCommand();
-            cmd.CommandText = ddl;
-            await cmd.ExecuteNonQueryAsync(token);
-
-            // Ensure created_at column exists (migration for tables created before this column was added)
-            await using var migrateCmd = conn.CreateCommand();
-            migrateCmd.CommandText = BuildAddMissingColumnsDdl(provider.Mapping);
-            await migrateCmd.ExecuteNonQueryAsync(token);
+            // Use Weasel SchemaMigration to create or update the document table
+            var table = new DocumentTable(provider.Mapping);
+            var migration = await SchemaMigration.DetermineAsync(conn, token, table);
+            await migrator.ApplyAllAsync(conn, migration, AutoCreate.CreateOrUpdate, ct: token);
 
             // Create custom indexes (computed columns + index)
+            // Computed columns are not modeled in Weasel, so they remain as supplementary DDL.
             // Each statement executed separately so computed columns are visible
-            // before filtered indexes reference them
+            // before filtered indexes reference them.
             foreach (var index in provider.Mapping.Indexes)
             {
                 foreach (var statement in index.ToDdlStatements(provider.Mapping))
@@ -147,106 +144,4 @@ public async Task EnsureTablesAsync(IEnumerable<DocumentProvider> providers, Can
             await EnsureTableAsync(provider, token);
         }
     }
-
-    private static string BuildHiloTableDdl(string schema)
-    {
-        return $"""
-            IF NOT EXISTS (SELECT 1 FROM sys.tables t
-                           JOIN sys.schemas s ON t.schema_id = s.schema_id
-                           WHERE s.name = '{schema}' AND t.name = 'pc_hilo')
-            BEGIN
-                IF SCHEMA_ID('{schema}') IS NULL
-                    EXEC('CREATE SCHEMA [{schema}]');
-
-                CREATE TABLE [{schema}].[pc_hilo] (
-                    entity_name varchar(250) NOT NULL PRIMARY KEY,
-                    hi_value bigint NOT NULL DEFAULT 0
-                );
-            END
-            """;
-    }
-
-    private static string BuildCreateTableDdl(DocumentMapping mapping)
-    {
-        var schema = mapping.DatabaseSchemaName;
-        var table = mapping.TableName;
-        var innerIdType = mapping.InnerIdType;
-        var idType = innerIdType == typeof(Guid) ? "uniqueidentifier"
-            : innerIdType == typeof(int) ? "int"
-            : innerIdType == typeof(long) ? "bigint"
-            : "varchar(250)";
-        var isConjoined = mapping.TenancyStyle == TenancyStyle.Conjoined;
-        var isSoftDelete = mapping.DeleteStyle == DeleteStyle.SoftDelete;
-        var softDeleteCols = isSoftDelete
-            ? @"
-                    is_deleted bit NOT NULL DEFAULT 0,
-                    deleted_at datetimeoffset NULL,"
-            : "";
-        var guidVersionCol = mapping.UseOptimisticConcurrency
-            ? @"
-                    guid_version uniqueidentifier NOT NULL DEFAULT NEWID(),"
-            : "";
-        var docTypeCol = mapping.IsHierarchy()
-            ? @"
-                    doc_type varchar(250) NOT NULL DEFAULT 'base',"
-            : "";
-
-        if (isConjoined)
-        {
-            return $@"
-                IF NOT EXISTS (SELECT 1 FROM sys.tables t
-                               JOIN sys.schemas s ON t.schema_id = s.schema_id
-                               WHERE s.name = '{schema}' AND t.name = '{table}')
-                BEGIN
-                    IF SCHEMA_ID('{schema}') IS NULL
-                        EXEC('CREATE SCHEMA [{schema}]');
-
-                    CREATE TABLE [{schema}].[{table}] (
-                        tenant_id varchar(250) NOT NULL,
-                        id {idType} NOT NULL,
-                        data nvarchar(max) NOT NULL,
-                        version int NOT NULL DEFAULT 1,
-                        last_modified datetimeoffset NOT NULL DEFAULT SYSDATETIMEOFFSET(),
-                        created_at datetimeoffset NOT NULL DEFAULT SYSDATETIMEOFFSET(),
-                        dotnet_type varchar(500) NULL,{docTypeCol}{softDeleteCols}{guidVersionCol}
-                        CONSTRAINT pk_{table} PRIMARY KEY (tenant_id, id)
-                    );
-                END";
-        }
-
-        return $@"
-            IF NOT EXISTS (SELECT 1 FROM sys.tables t
-                           JOIN sys.schemas s ON t.schema_id = s.schema_id
-                           WHERE s.name = '{schema}' AND t.name = '{table}')
-            BEGIN
-                IF SCHEMA_ID('{schema}') IS NULL
-                    EXEC('CREATE SCHEMA [{schema}]');
-
-                CREATE TABLE [{schema}].[{table}] (
-                    id {idType} NOT NULL PRIMARY KEY,
-                    data nvarchar(max) NOT NULL,
-                    version int NOT NULL DEFAULT 1,
-                    last_modified datetimeoffset NOT NULL DEFAULT SYSDATETIMEOFFSET(),
-                    created_at datetimeoffset NOT NULL DEFAULT SYSDATETIMEOFFSET(),
-                    dotnet_type varchar(500) NULL,{docTypeCol}{softDeleteCols}{guidVersionCol}
-                    tenant_id varchar(250) NOT NULL DEFAULT '*DEFAULT*'
-                );
-            END";
-    }
-
-    private static string BuildAddMissingColumnsDdl(DocumentMapping mapping)
-    {
-        var schema = mapping.DatabaseSchemaName;
-        var table = mapping.TableName;
-        return $"""
-            IF NOT EXISTS (SELECT 1 FROM sys.columns
-                           WHERE object_id = OBJECT_ID('[{schema}].[{table}]')
-                             AND name = 'created_at')
-            BEGIN
-                ALTER TABLE [{schema}].[{table}]
-                    ADD created_at datetimeoffset NOT NULL DEFAULT SYSDATETIMEOFFSET();
-            END
-            """;
-    }
-
 }

src/Polecat/Projections/Flattened/FlatTableProjection.cs

@@ -232,40 +232,15 @@ public ISubscriptionExecution BuildExecution(
 
     private async Task EnsureTableAsync(DocumentSessionBase session, CancellationToken cancellation)
     {
-        await using var cmd = new Microsoft.Data.SqlClient.SqlCommand();
-        cmd.CommandText = $"""
-            IF NOT EXISTS (SELECT * FROM sys.tables t
-                JOIN sys.schemas s ON t.schema_id = s.schema_id
-                WHERE s.name = '{Table.Identifier.Schema}' AND t.name = '{Table.Identifier.Name}')
-            BEGIN
-                {GenerateCreateTableDdl()}
-            END
-            """;
+        var migrator = new SqlServerMigrator();
+        var writer = new StringWriter();
+        Table.WriteCreateStatement(migrator, writer);
 
+        await using var cmd = new SqlCommand();
+        cmd.CommandText = writer.ToString();
         await session.ExecuteAsync(cmd, cancellation);
     }
 
-    private string GenerateCreateTableDdl()
-    {
-        var columns = Table.Columns.Select(c =>
-        {
-            var nullable = c.AllowNulls && !c.IsPrimaryKey ? "NULL" : "NOT NULL";
-            var identity = c.IsAutoNumber ? " IDENTITY(1,1)" : "";
-            return $"[{c.Name}] {c.Type}{identity} {nullable}";
-        });
-
-        var pkColumns = Table.PrimaryKeyColumns;
-        var pkConstraint = pkColumns.Count > 0
-            ? $", CONSTRAINT [PK_{Table.Identifier.Name}] PRIMARY KEY ({string.Join(", ", pkColumns.Select(c => $"[{c}]"))})"
-            : "";
-
-        return $"""
-            CREATE TABLE [{Table.Identifier.Schema}].[{Table.Identifier.Name}] (
-                {string.Join(",\n            ", columns)}{pkConstraint}
-            );
-            """;
-    }
-
     private static MemberInfo[]? GetMemberPath<T>(Expression<Func<T, object>> expression)
     {
         var body = expression.Body;

src/Polecat/Schema/Identity/Sequences/HiloSequence.cs

@@ -1,7 +1,10 @@
+using JasperFx;
 using Microsoft.Data.SqlClient;
 using Polly;
 using Polecat.Exceptions;
 using Polecat.Internal;
+using Weasel.Core;
+using Weasel.SqlServer;
 
 namespace Polecat.Schema.Identity.Sequences;
 
@@ -119,40 +122,24 @@ private async Task EnsureHiloTableAsync(SqlConnection conn, CancellationToken ct
     {
         if (_tableEnsured) return;
 
-        await using var cmd = conn.CreateCommand();
-        cmd.CommandText = HiloTableDdl();
-        await cmd.ExecuteNonQueryAsync(ct);
+        var table = new HiloTable(_schemaName);
+        var migrator = new SqlServerMigrator();
+        var migration = await SchemaMigration.DetermineAsync(conn, ct, table);
+        await migrator.ApplyAllAsync(conn, migration, AutoCreate.CreateOrUpdate, ct: ct);
         _tableEnsured = true;
     }
 
     private void EnsureHiloTableSync(SqlConnection conn)
     {
         if (_tableEnsured) return;
 
-        using var cmd = conn.CreateCommand();
-        cmd.CommandText = HiloTableDdl();
-        cmd.ExecuteNonQuery();
+        var table = new HiloTable(_schemaName);
+        var migrator = new SqlServerMigrator();
+        var migration = SchemaMigration.DetermineAsync(conn, table).GetAwaiter().GetResult();
+        migrator.ApplyAllAsync(conn, migration, AutoCreate.CreateOrUpdate).GetAwaiter().GetResult();
         _tableEnsured = true;
     }
 
-    private string HiloTableDdl()
-    {
-        return $"""
-            IF NOT EXISTS (SELECT 1 FROM sys.tables t
-                           JOIN sys.schemas s ON t.schema_id = s.schema_id
-                           WHERE s.name = '{_schemaName}' AND t.name = 'pc_hilo')
-            BEGIN
-                IF SCHEMA_ID('{_schemaName}') IS NULL
-                    EXEC('CREATE SCHEMA [{_schemaName}]');
-
-                CREATE TABLE [{_schemaName}].[pc_hilo] (
-                    entity_name varchar(250) NOT NULL PRIMARY KEY,
-                    hi_value bigint NOT NULL DEFAULT 0
-                );
-            END
-            """;
-    }
-
     private async Task<long> TryGetNextHiAsync(SqlConnection conn, CancellationToken ct)
     {
         // Read current hi_value

src/Polecat/Schema/Identity/Sequences/HiloTable.cs

@@ -0,0 +1,19 @@
+using Weasel.SqlServer;
+using Weasel.SqlServer.Tables;
+
+namespace Polecat.Schema.Identity.Sequences;
+
+/// <summary>
+///     Weasel table definition for pc_hilo — stores HiLo sequence counters for numeric ID generation.
+/// </summary>
+internal class HiloTable : Table
+{
+    public const string TableName = "pc_hilo";
+
+    public HiloTable(string schemaName)
+        : base(new SqlServerObjectName(schemaName, TableName))
+    {
+        AddColumn("entity_name", "varchar(250)").AsPrimaryKey().NotNull();
+        AddColumn("hi_value", "bigint").NotNull().DefaultValue(0);
+    }
+}

src/Polecat/Storage/DocumentFeatureSchema.cs

@@ -0,0 +1,37 @@
+using Polecat.Schema.Identity.Sequences;
+using Weasel.Core;
+using Weasel.Core.Migrations;
+using Weasel.SqlServer;
+
+namespace Polecat.Storage;
+
+/// <summary>
+///     Weasel feature schema that yields all document tables and the HiLo sequence table.
+///     Participates in ApplyAllConfiguredChangesToDatabaseAsync() for schema migration.
+/// </summary>
+internal class DocumentFeatureSchema : FeatureSchemaBase
+{
+    private readonly StoreOptions _options;
+
+    public DocumentFeatureSchema(StoreOptions options)
+        : base("Documents", new SqlServerMigrator())
+    {
+        _options = options;
+    }
+
+    public override Type StorageType => typeof(DocumentFeatureSchema);
+
+    protected override IEnumerable<ISchemaObject> schemaObjects()
+    {
+        // HiLo table first — numeric ID document types depend on it
+        if (_options.Providers.AllProviders.Any(p => p.Mapping.IsNumericId))
+        {
+            yield return new HiloTable(_options.DatabaseSchemaName);
+        }
+
+        foreach (var provider in _options.Providers.AllProviders)
+        {
+            yield return new DocumentTable(provider.Mapping);
+        }
+    }
+}