Skip to content

Models with custom materialization cannot be deleted #5761

@MikeWallis42

Description

@MikeWallis42

SQLMesh version: 0.227.1
sqlglot version: 27.28.1

When running sqlmesh janitor or sqlmesh destroy any model that has a custom materialization fails to be deleted because the custom materialization isn't loaded.

Traceback (most recent call last):
  File ".venv/lib/python3.12/site-packages/sqlmesh/utils/concurrency.py", line 69, in _process_node
    self.fn(node)
  File ".venv/lib/python3.12/site-packages/sqlmesh/utils/concurrency.py", line 172, in <lambda>
    lambda s_id: fn(snapshots_by_id[s_id]),
                 ^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/snapshot/evaluator.py", line 537, in <lambda>
    lambda s: self._cleanup_snapshot(
              ^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/snapshot/evaluator.py", line 1337, in _cleanup_snapshot
    evaluation_strategy = _evaluation_strategy(snapshot, adapter)
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/snapshot/evaluator.py", line 1683, in _evaluation_strategy
    _, klass = get_custom_materialization_type_or_raise(snapshot.custom_materialization)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/snapshot/evaluator.py", line 2918, in get_custom_materialization_type_or_raise
    types = get_custom_materialization_type(name, raise_errors=True)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/snapshot/evaluator.py", line 2900, in get_custom_materialization_type
    raise e
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/snapshot/evaluator.py", line 2897, in get_custom_materialization_type
    raise ConfigError(f"Materialization strategy with name '{name}' was not found.")
sqlmesh.utils.errors.ConfigError: Materialization strategy with name 'delsert' was not found.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File ".venv/bin/sqlmesh", line 10, in <module>
    sys.exit(cli())
             ^^^^^
  File ".venv/lib/python3.12/site-packages/click/core.py", line 1161, in __call__
    return self.main(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/click/core.py", line 1082, in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/click/core.py", line 1697, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/click/core.py", line 1443, in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/click/core.py", line 788, in invoke
    return __callback(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/click/decorators.py", line 33, in new_func
    return f(get_current_context(), *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/cli/__init__.py", line 29, in wrapper
    return handler(sqlmesh_context, lambda: func(*args, **kwargs))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/cli/__init__.py", line 51, in _debug_exception_handler
    return func()
           ^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/cli/__init__.py", line 29, in <lambda>
    return handler(sqlmesh_context, lambda: func(*args, **kwargs))
                                            ^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/analytics/__init__.py", line 82, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/cli/main.py", line 643, in janitor
    ctx.obj.run_janitor(ignore_ttl, **kwargs)
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/analytics/__init__.py", line 110, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/context.py", line 872, in run_janitor
    self._run_janitor(ignore_ttl)
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/context.py", line 2868, in _run_janitor
    delete_expired_snapshots(
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/janitor.py", line 168, in delete_expired_snapshots
    snapshot_evaluator.cleanup(
  File ".venv/lib/python3.12/site-packages/sqlmesh/core/snapshot/evaluator.py", line 535, in cleanup
    concurrent_apply_to_snapshots(
  File ".venv/lib/python3.12/site-packages/sqlmesh/utils/concurrency.py", line 170, in concurrent_apply_to_snapshots
    return concurrent_apply_to_dag(
           ^^^^^^^^^^^^^^^^^^^^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/utils/concurrency.py", line 212, in concurrent_apply_to_dag
    ).run()
      ^^^^^
  File ".venv/lib/python3.12/site-packages/sqlmesh/utils/concurrency.py", line 64, in run
    self._finished_future.result()
  File ".local/share/uv/python/cpython-3.12.11-macos-aarch64-none/lib/python3.12/concurrent/futures/_base.py", line 456, in result
    return self.__get_result()
           ^^^^^^^^^^^^^^^^^^^
  File "local/share/uv/python/cpython-3.12.11-macos-aarch64-none/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
sqlmesh.utils.concurrency.NodeExecutionFailedError: Execution failed for node SnapshotId<"iceberg"."explorer_ipcv"."pre_care_record_range_v1": 2524880239>

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions