diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d3beea4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM fkrull/multi-python as build +ENV PYTHONUNBUFFERED 1 +ARG PIP_INDEX_URL +ENV PIP_INDEX_URL ${PIP_INDEX_URL} +RUN pip3 --no-cache install --upgrade pip +COPY setup.py . +COPY psqlextra/_version.py psqlextra/_version.py +COPY README.md . +RUN pip3 install .[test] .[analysis] --no-cache-dir --no-cache --prefix /python-packages --no-warn-script-location + +FROM fkrull/multi-python +ENV PROJECT_DIR /project +WORKDIR $PROJECT_DIR +ENV PYTHONUNBUFFERED 1 +COPY --from=build /python-packages /usr/local +COPY . . \ No newline at end of file diff --git a/Dockerfile-py39-DJ-32 b/Dockerfile-py39-DJ-32 new file mode 100644 index 0000000..04416e2 --- /dev/null +++ b/Dockerfile-py39-DJ-32 @@ -0,0 +1,17 @@ +FROM python:3.9-bullseye as build +ENV PYTHONUNBUFFERED 1 +ARG PIP_INDEX_URL +ENV PIP_INDEX_URL ${PIP_INDEX_URL} +RUN pip3 --no-cache install --upgrade pip +COPY setup.py . +COPY psqlextra/_version.py psqlextra/_version.py +COPY README.md . +RUN pip3 install .[test] .[analysis] --no-cache-dir --no-cache --prefix /python-packages --no-warn-script-location +RUN pip3 install django==3.2 --no-cache-dir --no-cache --prefix /python-packages --no-warn-script-location + +FROM python:3.9-bullseye +ENV PROJECT_DIR /django-postgres-extra-fork +WORKDIR $PROJECT_DIR +ENV PYTHONUNBUFFERED 1 +COPY --from=build /python-packages /usr/local +COPY . . \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..34fb240 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +test: + docker-compose up --build tests +test-fast: + docker-compose up --build quick-tests \ No newline at end of file diff --git a/README.md b/README.md index 603fef8..fb61c43 100644 --- a/README.md +++ b/README.md @@ -110,3 +110,8 @@ These are just for local development. CI for code analysis etc runs against thes 7. Auto-format code, sort imports and auto-fix linting errors: λ poe fix + +### Running tests with Docker Compose (ASI-specific) + + λ docker compose -f docker-compose.tests.yml run --rm tests + diff --git a/docker-compose.tests.yml b/docker-compose.tests.yml new file mode 100644 index 0000000..d401f7f --- /dev/null +++ b/docker-compose.tests.yml @@ -0,0 +1,42 @@ +version: "3.9" + +x-test-env: &test-env + DATABASE_URL: postgres://psqlextra:psqlextra@postgres:5432/psqlextra + DATABASE_IN_CONTAINER: "true" + DJANGO_SETTINGS_MODULE: settings + PGPASSWORD: psqlextra + PYTHONDONTWRITEBYTECODE: "1" + PYTHONUNBUFFERED: "1" + +services: + postgres: + image: postgres:16.0 + environment: + POSTGRES_DB: psqlextra + POSTGRES_USER: psqlextra + POSTGRES_PASSWORD: psqlextra + healthcheck: + test: ["CMD-SHELL", "pg_isready -U psqlextra -d psqlextra -h localhost"] + interval: 5s + timeout: 5s + retries: 10 + ports: + - "5435:5432" + + tests: + build: + context: . + dockerfile: docker/tests/Dockerfile + args: + PYTHON_VERSION: ${PYTHON_VERSION:-3.12} + DEBIAN_DIST: ${DEBIAN_DIST:-bookworm} + environment: + <<: *test-env + TOX_ARGS: ${TOX_ARGS:--e py312-dj52-psycopg32} + depends_on: + postgres: + condition: service_healthy + volumes: + - .:/workspace + working_dir: /workspace + command: bash -lc "tox ${TOX_ARGS}" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3b499a6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,42 @@ +version: "3.7" + +services: + postgres: + image: postgis/postgis:14-3.2 + ports: + - "5435:5432" + volumes: + - pgdata:/var/lib/postgresql/data/:delegated + environment: + - POSTGRES_HOST_AUTH_METHOD=trust + - POSTGRES_DB=psqlextra + healthcheck: + test: "pg_isready -h localhost -p 5432 -q -U postgres" + interval: 3s + timeout: 5s + retries: 3 + + + tests: + build: + context: ./ + depends_on: + - postgres + command: + "tox" + + quick-tests: + build: + dockerfile: Dockerfile-py39-DJ-32 + context: ./ + depends_on: + - postgres + environment: + - DJANGO_SETTINGS_MODULE=settings + volumes: + - ./prof:/django-postgres-extra-fork/prof/ + command: + "pytest -s --reuse-db --durations=10" + +volumes: + pgdata: diff --git a/psqlextra/_version.py b/psqlextra/_version.py index e8733fa..bdb0729 100644 --- a/psqlextra/_version.py +++ b/psqlextra/_version.py @@ -1 +1 @@ -__version__ = "2.0.9rc4" +__version__ = "2.0.9dev2" diff --git a/psqlextra/backend/schema.py b/psqlextra/backend/schema.py index b8dd0a7..ab7cc71 100644 --- a/psqlextra/backend/schema.py +++ b/psqlextra/backend/schema.py @@ -78,10 +78,20 @@ class PostgresSchemaEditor(SchemaEditor): sql_add_range_partition = ( "CREATE TABLE %s PARTITION OF %s FOR VALUES FROM (%s) TO (%s)" ) + sql_create_unattached_partition = ( + "CREATE TABLE %s (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS)" + ) + sql_attach_range_partition = ( + "ALTER TABLE %s ATTACH PARTITION %s FOR VALUES FROM (%s) TO (%s)" + ) sql_add_list_partition = ( "CREATE TABLE %s PARTITION OF %s FOR VALUES IN (%s)" ) sql_delete_partition = "DROP TABLE %s" + sql_detach_partition = "ALTER TABLE %s DETACH PARTITION %s" + sql_detach_partition_concurrently = ( + "ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY" + ) sql_table_comment = "COMMENT ON TABLE %s IS %s" side_effects: List[DatabaseSchemaEditor] = [ @@ -89,7 +99,7 @@ class PostgresSchemaEditor(SchemaEditor): cast(DatabaseSchemaEditor, HStoreRequiredSchemaEditorSideEffect()), ] - def __init__(self, connection, collect_sql=False, atomic=True): + def __init__(self, connection, collect_sql=False, atomic=False): super().__init__(connection, collect_sql, atomic) for side_effect in self.side_effects: @@ -98,6 +108,47 @@ def __init__(self, connection, collect_sql=False, atomic=True): self.deferred_sql = [] self.introspection = PostgresIntrospection(self.connection) + self._clone_atomic: Optional[transaction.Atomic] = None + + def __enter__(self): + return super().__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + self._finish_clone_atomic(exc_type, exc_value, traceback) + return super().__exit__(exc_type, exc_value, traceback) + + def _ensure_clone_atomic(self) -> None: + """Starts an atomic block when cloning schema objects requires holding + locks over multiple statements. + + When Django already manages a transaction (e.g. during a migration), we + reuse that transaction and avoid nesting another atomic block. We only + open our own transaction when the caller operates in autocommit mode. + """ + + if self.connection.in_atomic_block or self._clone_atomic is not None: + return + + atomic_ctx = transaction.atomic(using=self.connection.alias) + atomic_ctx.__enter__() + self._clone_atomic = atomic_ctx + + def _finish_clone_atomic(self, exc_type, exc_value, traceback) -> None: + """Completes any atomic block started by `_ensure_clone_atomic`.""" + + if not self._clone_atomic: + return + + atomic_ctx = self._clone_atomic + self._clone_atomic = None + atomic_ctx.__exit__(exc_type, exc_value, traceback) + + def _ensure_autocommit(self) -> None: + """Makes sure there is no outstanding atomic block created by + `_ensure_clone_atomic` before running statements that must execute + outside a transaction.""" + + self._finish_clone_atomic(None, None, None) def create_schema(self, name: str) -> None: """Creates a Postgres schema.""" @@ -162,6 +213,8 @@ def clone_model_structure_to_schema( quoted_table_fqn = f"{quoted_schema_name}.{quoted_table_name}" + self._ensure_clone_atomic() + self.execute( self.sql_create_table % { @@ -176,7 +229,9 @@ def clone_model_structure_to_schema( # not copy the sequences into the new table. We do it manually. if django.VERSION < (4, 1): with self.connection.cursor() as cursor: - sequences = self.introspection.get_sequences(cursor, table_name) + sequences = self.introspection.get_sequences( + cursor, table_name + ) for sequence in sequences: if sequence["table"] != table_name: @@ -239,6 +294,8 @@ def clone_model_constraints_and_indexes_to_schema( resides. """ + self._ensure_clone_atomic() + with postgres_prepend_local_search_path( [schema_name], using=self.connection.alias ): @@ -374,6 +431,8 @@ def clone_model_foreign_keys_to_schema( resides. """ + self._ensure_clone_atomic() + constraint_names = self._constraint_names(model, foreign_key=True) # type: ignore[attr-defined] with postgres_prepend_local_search_path( @@ -529,6 +588,9 @@ def refresh_materialized_view_model( else self.sql_refresh_materialized_view ) + if concurrently: + self._ensure_autocommit() + sql = sql_template % self.quote_name(model._meta.db_table) self.execute(sql) @@ -674,6 +736,37 @@ def delete_partitioned_model(self, model: Type[Model]) -> None: return self.delete_model(model) + def add_range_partition_deferred( + self, + model: Model, + name: str, + from_values: Any, + to_values: Any, + comment: Optional[str] = None, + ) -> None: + # asserts the model is a model set up for partitioning + self._partitioning_properties_for_model(model) + + table_name = self.create_partition_table_name(model, name) + + sql_create_unattached = self.sql_create_unattached_partition % ( + self.quote_name(table_name), + self.quote_name(model._meta.db_table), + ) + sql_attach_partition = self.sql_attach_range_partition % ( + self.quote_name(model._meta.db_table), + self.quote_name(table_name), + "%s", + "%s", + ) + + with transaction.atomic(): + self.execute(sql_create_unattached) + self.execute(sql_attach_partition, (from_values, to_values)) + + if comment: + self.set_comment_on_table(table_name, comment) + def add_range_partition( self, model: Type[Model], @@ -859,6 +952,25 @@ def delete_partition(self, model: Type[Model], name: str) -> None: ) self.execute(sql) + def detach_partition(self, model: Model, name: str) -> None: + """Detaches the partition with the specified name.""" + + sql = self.sql_detach_partition % ( + self.quote_name(model._meta.db_table), + self.quote_name(self.create_partition_table_name(model, name)), + ) + self.execute(sql) + + def detach_partition_concurrently(self, model: Model, name: str) -> None: + """Detaches concurrently the partition with the specified name.""" + + self._ensure_autocommit() + sql = self.sql_detach_partition_concurrently % ( + self.quote_name(model._meta.db_table), + self.quote_name(self.create_partition_table_name(model, name)), + ) + self.execute(sql) + def alter_db_table( self, model: Type[Model], old_db_table: str, new_db_table: str ) -> None: diff --git a/psqlextra/management/commands/pgpartition.py b/psqlextra/management/commands/pgpartition.py index ca62166..6154b60 100644 --- a/psqlextra/management/commands/pgpartition.py +++ b/psqlextra/management/commands/pgpartition.py @@ -13,7 +13,11 @@ class Command(BaseCommand): """Create new partitions and delete old ones according to the configured partitioning strategies.""" - help = "Create new partitions and delete old ones using the configured partitioning manager. The PSQLEXTRA_PARTITIONING_MANAGER setting must be configured." + help = ( + "Create new partitions and delete old ones using the configured " + "partitioning manager. The PSQLEXTRA_PARTITIONING_MANAGER setting must " + "be configured." + ) def add_arguments(self, parser): parser.add_argument( @@ -41,6 +45,22 @@ def add_arguments(self, parser): default="default", ) + parser.add_argument( + "--skip-create", + action="https://github.com/store_true", + help="Do not create partitions.", + required=False, + default=False, + ) + + parser.add_argument( + "--skip-delete", + action="https://github.com/store_true", + help="Do not delete partitions.", + required=False, + default=False, + ) + parser.add_argument( "--model-names", "-m", @@ -50,18 +70,18 @@ def add_arguments(self, parser): ) parser.add_argument( - "--skip-create", - action="https://github.com/store_true", - help="Do not create partitions.", + "--detach", + help="Detach partitions", required=False, - default=False, + choices=["no", "sequentially", "concurrently"], + default="no", ) parser.add_argument( - "--skip-delete", - action="https://github.com/store_true", - help="Do not delete partitions.", + "--defer-attach", + help="First create, then attach partitions", required=False, + action="https://github.com/store_true", default=False, ) @@ -73,6 +93,8 @@ def handle( # type: ignore[override] skip_create: bool, skip_delete: bool, model_names: Optional[List[str]] = None, + detach: str = "no", + defer_attach: bool = False, *args, **kwargs, ): @@ -82,12 +104,20 @@ def handle( # type: ignore[override] skip_create=skip_create, skip_delete=skip_delete, model_names=model_names, + detach=detach, using=using, + deferred_attach=defer_attach, ) creations_count = len(plan.creations) deletions_count = len(plan.deletions) - if creations_count == 0 and deletions_count == 0: + deferred_creations_count = len(plan.deferred_creations) + + if ( + creations_count == 0 + and deletions_count == 0 + and deferred_creations_count == 0 + ): print("Nothing to be done.") return diff --git a/psqlextra/models/partitioned.py b/psqlextra/models/partitioned.py index 3a20677..2861e0d 100644 --- a/psqlextra/models/partitioned.py +++ b/psqlextra/models/partitioned.py @@ -1,7 +1,5 @@ from typing import Iterable, List, Optional, Tuple -import django - from django.core.exceptions import ImproperlyConfigured from django.db import models from django.db.models.base import ModelBase @@ -29,11 +27,9 @@ def __new__(cls, name, bases, attrs, **kwargs): partitioning_method = getattr(partitioning_meta_class, "method", None) partitioning_key = getattr(partitioning_meta_class, "key", None) + special = getattr(partitioning_meta_class, "special", None) - if django.VERSION >= (5, 2): - for base in bases: - cls._delete_auto_created_fields(base) - + if special: cls._create_primary_key(attrs, partitioning_key) patitioning_meta = PostgresPartitionedModelOptions( @@ -46,57 +42,23 @@ def __new__(cls, name, bases, attrs, **kwargs): return new_class @classmethod - def _create_primary_key( - cls, attrs, partitioning_key: Optional[List[str]] - ) -> None: + def _create_primary_key(cls, attrs, partitioning_key: Optional[List[str]]): from django.db.models.fields.composite import CompositePrimaryKey - # Find any existing primary key the user might have declared. - # - # If it is a composite primary key, we will do nothing and - # keep it as it is. You're own your own. pk = cls._find_primary_key(attrs) if pk and isinstance(pk[1], CompositePrimaryKey): return - # Create an `id` field (auto-incrementing) if there is no - # primary key yet. - # - # This matches standard Django behavior. if not pk: attrs["id"] = attrs.get("id") or cls._create_auto_field(attrs) pk_fields = ["id"] else: pk_fields = [pk[0]] - partitioning_keys = ( - partitioning_key - if isinstance(partitioning_key, list) - else list(filter(None, [partitioning_key])) - ) - - unique_pk_fields = set(pk_fields + (partitioning_keys or [])) + unique_pk_fields = set(pk_fields + (partitioning_key or [])) if len(unique_pk_fields) <= 1: - if "id" in attrs: - attrs["id"].primary_key = True return - # You might have done something like this: - # - # id = models.AutoField(primary_key=True) - # pk = CompositePrimaryKey("id", "timestamp") - # - # The `primary_key` attribute has to be removed - # from the `id` field in the example above to - # avoid having two primary keys. - # - # Without this, the generated schema will - # have two primary keys, which is an error. - for field in attrs.values(): - is_pk = getattr(field, "primary_key", False) - if is_pk: - field.primary_key = False - auto_generated_pk = CompositePrimaryKey(*sorted(unique_pk_fields)) attrs["pk"] = auto_generated_pk @@ -106,7 +68,7 @@ def _create_auto_field(cls, attrs): meta_class = attrs.get("Meta", None) pk_class = Options(meta_class, app_label)._get_default_pk_class() - return pk_class(verbose_name="ID", auto_created=True) + return pk_class(verbose_name="ID", primary_key=True, auto_created=True) @classmethod def _find_primary_key(cls, attrs) -> Optional[Tuple[str, models.Field]]: @@ -139,7 +101,6 @@ def _find_primary_key(cls, attrs) -> Optional[Tuple[str, models.Field]]: 3. There is no primary key. """ - from django.db.models.fields.composite import CompositePrimaryKey fields = { @@ -199,29 +160,6 @@ def _find_primary_key(cls, attrs) -> Optional[Tuple[str, models.Field]]: return sorted_fields_marked_as_pk[0] - @classmethod - def _delete_auto_created_fields(cls, model: models.Model): - """Base classes might be injecting an auto-generated `id` field before - we even have the chance of doing this ourselves. - - Delete any auto generated fields from the base class so that we - can declare our own. If there is no auto-generated field, one - will be added anyways by our own logic - """ - - fields = model._meta.local_fields + model._meta.local_many_to_many - for field in fields: - auto_created = getattr(field, "auto_created", False) - if auto_created: - if field in model._meta.local_fields: - model._meta.local_fields.remove(field) - - if field in model._meta.fields: - model._meta.fields.remove(field) # type: ignore [attr-defined] - - if hasattr(model, field.name): - delattr(model, field.name) - class PostgresPartitionedModel( PostgresModel, metaclass=PostgresPartitionedModelMeta diff --git a/psqlextra/partitioning/delete_on_condition_strategy.py b/psqlextra/partitioning/delete_on_condition_strategy.py new file mode 100644 index 0000000..c95a43d --- /dev/null +++ b/psqlextra/partitioning/delete_on_condition_strategy.py @@ -0,0 +1,26 @@ +from typing import Callable, Generator + +from psqlextra.partitioning import ( + PostgresPartition, + PostgresPartitioningStrategy, +) + + +class PostgresDeleteOnConditionPartitioningStrategy( + PostgresPartitioningStrategy +): + def __init__( + self, + delegate: PostgresPartitioningStrategy, + delete_condition: Callable[[PostgresPartition], bool], + ): + self._delegate = delegate + self._delete_condition = delete_condition + + def to_create(self,) -> Generator[PostgresPartition, None, None]: + return self._delegate.to_create() + + def to_delete(self,) -> Generator[PostgresPartition, None, None]: + for partition in self._delegate.to_delete(): + setattr(partition, 'to_be_deleted', self._delete_condition(partition)) + yield partition diff --git a/psqlextra/partitioning/manager.py b/psqlextra/partitioning/manager.py index 01bac3b..525c29c 100644 --- a/psqlextra/partitioning/manager.py +++ b/psqlextra/partitioning/manager.py @@ -27,6 +27,8 @@ def plan( skip_delete: bool = False, model_names: Optional[List[str]] = None, using: Optional[str] = None, + detach: Optional[str] = None, + deferred_attach: Optional[bool] = None ) -> PostgresPartitioningPlan: """Plans which partitions should be deleted/created. @@ -71,6 +73,8 @@ def plan( skip_create=skip_create, skip_delete=skip_delete, using=using, + detach=detach, + defer_attach=deferred_attach, ) if not model_plan: continue @@ -94,6 +98,8 @@ def _plan_for_config( skip_create: bool = False, skip_delete: bool = False, using: Optional[str] = None, + detach: Optional[str] = None, + defer_attach: Optional[bool] = None, ) -> Optional[PostgresModelPartitioningPlan]: """Creates a partitioning plan for one partitioning config.""" @@ -103,26 +109,38 @@ def _plan_for_config( model_plan = PostgresModelPartitioningPlan(config) if not skip_create: - for partition in config.strategy.to_create(): - if table.partition_by_name(name=partition.name()): - continue - - model_plan.creations.append(partition) - - if not skip_delete: - for partition in config.strategy.to_delete(): - introspected_partition = table.partition_by_name( - name=partition.name() - ) - if not introspected_partition: - break - - if introspected_partition.comment != AUTO_PARTITIONED_COMMENT: - continue + if not defer_attach: + for partition in config.strategy.to_create(): + if table.partition_by_name(name=partition.name()): + continue + + model_plan.creations.append(partition) + else: + for partition in config.strategy.to_create(): + if table.partition_by_name(name=partition.name()): + continue + + model_plan.deferred_creations.append(partition) + + for partition in config.strategy.to_delete(): + introspected_partition = table.partition_by_name( + name=partition.name() + ) + if not introspected_partition: + break + if (introspected_partition.comment != AUTO_PARTITIONED_COMMENT + or not getattr(partition, "to_be_deleted", True)): + continue + if not skip_delete: model_plan.deletions.append(partition) - if len(model_plan.creations) == 0 and len(model_plan.deletions) == 0: + if detach == "concurrently": + model_plan.concurrent_detachements.append(partition) + elif detach == "sequentially": + model_plan.detachements.append(partition) + + if len(model_plan.creations) == 0 and len(model_plan.deferred_creations) == 0 and len(model_plan.deletions) == 0: return None return model_plan diff --git a/psqlextra/partitioning/partition.py b/psqlextra/partitioning/partition.py index 4c13fda..dea3286 100644 --- a/psqlextra/partitioning/partition.py +++ b/psqlextra/partitioning/partition.py @@ -18,6 +18,7 @@ def create( model: Type[PostgresPartitionedModel], schema_editor: PostgresSchemaEditor, comment: Optional[str] = None, + defer_attach: bool = False, ) -> None: """Creates this partition in the database.""" @@ -29,6 +30,21 @@ def delete( ) -> None: """Deletes this partition from the database.""" + @abstractmethod + def detach( + self, + model: PostgresPartitionedModel, + schema_editor: PostgresSchemaEditor, + concurrently: bool = False, + ) -> None: + """Detaches this partition from the database.""" + if concurrently: + schema_editor.detach_partition_concurrently( + model=model, name=self.name() + ) + else: + schema_editor.detach_partition(model=model, name=self.name()) + def deconstruct(self) -> dict: """Deconstructs this partition into a dict of attributes/fields.""" diff --git a/psqlextra/partitioning/plan.py b/psqlextra/partitioning/plan.py index 301b424..7a8c740 100644 --- a/psqlextra/partitioning/plan.py +++ b/psqlextra/partitioning/plan.py @@ -21,6 +21,11 @@ class PostgresModelPartitioningPlan: config: PostgresPartitioningConfig creations: List[PostgresPartition] = field(default_factory=list) + deferred_creations: List[PostgresPartition] = field(default_factory=list) + detachements: List[PostgresPartition] = field(default_factory=list) + concurrent_detachements: List[PostgresPartition] = field( + default_factory=list + ) deletions: List[PostgresPartition] = field(default_factory=list) def apply(self, using: Optional[str]) -> None: @@ -36,25 +41,61 @@ def apply(self, using: Optional[str]) -> None: connection = connections[using or "default"] - with transaction.atomic(): + if self.creations: + with transaction.atomic(): + with connection.schema_editor() as schema_editor: + editor = cast("PostgresSchemaEditor", schema_editor) + for partition in self.creations: + partition.create( + self.config.model, + editor, + comment=AUTO_PARTITIONED_COMMENT, + ) + + if self.deferred_creations: + with transaction.atomic(): + with connection.schema_editor() as schema_editor: + editor = cast("PostgresSchemaEditor", schema_editor) + for partition in self.deferred_creations: + partition.create( + self.config.model, + editor, + comment=AUTO_PARTITIONED_COMMENT, + defer_attach=True, + ) + + if self.detachements: + with transaction.atomic(): + with connection.schema_editor() as schema_editor: + editor = cast("PostgresSchemaEditor", schema_editor) + for partition in self.detachements: + partition.detach( + self.config.model, + editor, + concurrently=False, + ) + + if self.concurrent_detachements: with connection.schema_editor() as schema_editor: - for partition in self.creations: - partition.create( + editor = cast("PostgresSchemaEditor", schema_editor) + for partition in self.concurrent_detachements: + partition.detach( self.config.model, - cast("PostgresSchemaEditor", schema_editor), - comment=AUTO_PARTITIONED_COMMENT, + editor, + concurrently=True, ) - for partition in self.deletions: - partition.delete( - self.config.model, - cast("PostgresSchemaEditor", schema_editor), - ) + if self.deletions: + with transaction.atomic(): + with connection.schema_editor() as schema_editor: + editor = cast("PostgresSchemaEditor", schema_editor) + for partition in self.deletions: + partition.delete(self.config.model, editor) def print(self) -> None: """Prints this model plan to the terminal in a readable format.""" - print(f"{self.config.model.__name__}: ") + print(f"{self.config.model.__name__}:") for partition in self.deletions: print(" - %s" % partition.name()) @@ -66,6 +107,21 @@ def print(self) -> None: for key, value in partition.deconstruct().items(): print(f" {key}: {value}") + for partition in self.deferred_creations: + print(" + %s (deferred attach)" % partition.name()) + for key, value in partition.deconstruct().items(): + print(f" {key}: {value}") + + for partition in self.detachements: + print(" ~ %s (detached sequentially)" % partition.name()) + for key, value in partition.deconstruct().items(): + print(f" {key}: {value}") + + for partition in self.concurrent_detachements: + print(" ~ %s (detached concurrently)" % partition.name()) + for key, value in partition.deconstruct().items(): + print(f" {key}: {value}") + @dataclass class PostgresPartitioningPlan: @@ -78,17 +134,27 @@ def creations(self) -> List[PostgresPartition]: """Gets a complete flat list of the partitions that are going to be created.""" - creations = [] + creations: List[PostgresPartition] = [] for model_plan in self.model_plans: creations.extend(model_plan.creations) return creations + @property + def deferred_creations(self) -> List[PostgresPartition]: + """Gets a complete flat list of the partitions that are going to be + created.""" + + deferred_creations: List[PostgresPartition] = [] + for model_plan in self.model_plans: + deferred_creations.extend(model_plan.deferred_creations) + return deferred_creations + @property def deletions(self) -> List[PostgresPartition]: """Gets a complete flat list of the partitions that are going to be deleted.""" - deletions = [] + deletions: List[PostgresPartition] = [] for model_plan in self.model_plans: deletions.extend(model_plan.deletions) return deletions @@ -108,9 +174,14 @@ def print(self) -> None: create_count = len(self.creations) delete_count = len(self.deletions) + deferred_creations_count = len(self.deferred_creations) print(f"{delete_count} partitions will be deleted") print(f"{create_count} partitions will be created") + if deferred_creations_count: + print( + f"{deferred_creations_count} partitions will be created and attached later" + ) __all__ = ["PostgresPartitioningPlan", "PostgresModelPartitioningPlan"] diff --git a/psqlextra/partitioning/range_partition.py b/psqlextra/partitioning/range_partition.py index a2f3e82..812ece2 100644 --- a/psqlextra/partitioning/range_partition.py +++ b/psqlextra/partitioning/range_partition.py @@ -26,14 +26,24 @@ def create( model: Type[PostgresPartitionedModel], schema_editor: PostgresSchemaEditor, comment: Optional[str] = None, + defer_attach: bool = False, ) -> None: - schema_editor.add_range_partition( - model=model, - name=self.name(), - from_values=self.from_values, - to_values=self.to_values, - comment=comment, - ) + if not defer_attach: + schema_editor.add_range_partition( + model=model, + name=self.name(), + from_values=self.from_values, + to_values=self.to_values, + comment=comment, + ) + else: + schema_editor.add_range_partition_deferred( + model=model, + name=self.name(), + from_values=self.from_values, + to_values=self.to_values, + comment=comment, + ) def delete( self, diff --git a/psqlextra/partitioning/shorthands.py b/psqlextra/partitioning/shorthands.py index f263e36..fc5c7d4 100644 --- a/psqlextra/partitioning/shorthands.py +++ b/psqlextra/partitioning/shorthands.py @@ -55,12 +55,17 @@ def partition_by_current_time( a delete/cleanup run. name_format: - The datetime format which is being passed to datetime.strftime - to generate the partition name. + The datetime format passed to ``datetime.strftime`` + to generate the partition name. Defaults to a + sensible format per size unit. """ size = PostgresTimePartitionSize( - years=years, months=months, weeks=weeks, days=days, hours=hours + years=years, + months=months, + weeks=weeks, + days=days, + hours=hours, ) return PostgresPartitioningConfig( diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_auto_confirm[y].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_auto_confirm[y].json index 664538a..a3cab58 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_auto_confirm[y].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_auto_confirm[y].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nOperations applied.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nOperations applied.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_auto_confirm[yes].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_auto_confirm[yes].json index 664538a..a3cab58 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_auto_confirm[yes].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_auto_confirm[yes].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nOperations applied.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nOperations applied.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[capital_n].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[capital_n].json index f1c2aa6..fc859ff 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[capital_n].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[capital_n].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[capital_no].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[capital_no].json index f1c2aa6..fc859ff 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[capital_no].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[capital_no].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[n].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[n].json index f1c2aa6..fc859ff 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[n].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[n].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[no].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[no].json index f1c2aa6..fc859ff 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[no].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[no].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[title_no].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[title_no].json index f1c2aa6..fc859ff 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[title_no].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_no[title_no].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operation aborted.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[capital_y].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[capital_y].json index 530f6bd..b2c5cab 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[capital_y].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[capital_y].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operations applied.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operations applied.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[capital_yes].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[capital_yes].json index 530f6bd..b2c5cab 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[capital_yes].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[capital_yes].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operations applied.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operations applied.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[y].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[y].json index 530f6bd..b2c5cab 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[y].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[y].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operations applied.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operations applied.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[yes].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[yes].json index 530f6bd..b2c5cab 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[yes].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_confirm_yes[yes].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operations applied.\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\nDo you want to proceed? (y/N) Operations applied.\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_dry_run[d].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_dry_run[d].json index 6b67fa9..fbe4fb2 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_dry_run[d].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_dry_run[d].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\n" diff --git a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_dry_run[dry].json b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_dry_run[dry].json index 6b67fa9..fbe4fb2 100644 --- a/tests/__snapshots__/test_management_command_partition/test_management_command_partition_dry_run[dry].json +++ b/tests/__snapshots__/test_management_command_partition/test_management_command_partition_dry_run[dry].json @@ -1 +1 @@ -"test: \n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\n" +"test:\n - tobedeleted\n + tobecreated\n\n1 partitions will be deleted\n1 partitions will be created\n" diff --git a/tests/db_introspection.py b/tests/db_introspection.py index 285cd0e..744aedc 100644 --- a/tests/db_introspection.py +++ b/tests/db_introspection.py @@ -4,19 +4,28 @@ This makes test code less verbose and easier to read/write. """ -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager from typing import Optional -from django.db import connection +from django.db import connection, transaction from psqlextra.settings import postgres_set_local @contextmanager def introspect(schema_name: Optional[str] = None): - with postgres_set_local(search_path=schema_name or None): - with connection.cursor() as cursor: - yield connection.introspection, cursor + needs_atomic = not connection.in_atomic_block + + with ExitStack() as stack: + if needs_atomic: + stack.enter_context(transaction.atomic(using=connection.alias)) + + stack.enter_context( + postgres_set_local(search_path=schema_name or None) + ) + cursor = stack.enter_context(connection.cursor()) + + yield connection.introspection, cursor def table_names( diff --git a/tests/test_partitioned_model.py b/tests/test_partitioned_model.py index 55c6651..f4fc3c6 100644 --- a/tests/test_partitioned_model.py +++ b/tests/test_partitioned_model.py @@ -85,11 +85,11 @@ def test_partitioned_model_key_option_none(): def test_partitioned_model_custom_composite_primary_key_with_auto_field(): model = define_fake_partitioned_model( fields={ - "auto_id": models.AutoField(primary_key=True), + "auto_id": models.AutoField(), "my_custom_pk": models.CompositePrimaryKey("auto_id", "timestamp"), "timestamp": models.DateTimeField(), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) assert isinstance(model._meta.pk, models.CompositePrimaryKey) @@ -108,7 +108,7 @@ def test_partitioned_model_custom_composite_primary_key_with_id_field(): "my_custom_pk": models.CompositePrimaryKey("id", "timestamp"), "timestamp": models.DateTimeField(), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) assert isinstance(model._meta.pk, models.CompositePrimaryKey) @@ -127,7 +127,7 @@ def test_partitioned_model_custom_composite_primary_key_named_id(): "id": models.CompositePrimaryKey("other_field", "timestamp"), "timestamp": models.DateTimeField(), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) assert isinstance(model._meta.pk, models.CompositePrimaryKey) @@ -147,7 +147,7 @@ def test_partitioned_model_field_named_pk_not_composite_not_primary(): "id": models.CompositePrimaryKey("other_field", "timestamp"), "timestamp": models.DateTimeField(), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) @@ -162,7 +162,7 @@ def test_partitioned_model_field_named_pk_not_composite(): "pk": models.AutoField(primary_key=True), "timestamp": models.DateTimeField(), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) @@ -179,7 +179,7 @@ def test_partitioned_model_field_multiple_pks(): "timestamp": models.DateTimeField(), "real_pk": models.CompositePrimaryKey("id", "timestamp"), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) @@ -192,7 +192,7 @@ def test_partitioned_model_no_pk_defined(): fields={ "timestamp": models.DateTimeField(), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) assert isinstance(model._meta.pk, models.CompositePrimaryKey) @@ -203,7 +203,7 @@ def test_partitioned_model_no_pk_defined(): assert id_field.name == "id" assert id_field.column == "id" assert isinstance(id_field, models.AutoField) - assert id_field.primary_key is False + assert id_field.primary_key is True @pytest.mark.skipif( @@ -217,7 +217,7 @@ def test_partitioned_model_composite_primary_key(): "pk": models.CompositePrimaryKey("id", "timestamp"), "timestamp": models.DateTimeField(), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) assert isinstance(model._meta.pk, models.CompositePrimaryKey) @@ -234,7 +234,7 @@ def test_partitioned_model_composite_primary_key_foreign_key(): fields={ "timestamp": models.DateTimeField(), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) define_fake_model( @@ -255,7 +255,7 @@ def test_partitioned_model_custom_composite_primary_key_foreign_key(): "timestamp": models.DateTimeField(), "custom": models.CompositePrimaryKey("id", "timestamp"), }, - partitioning_options=dict(key=["timestamp"]), + partitioning_options=dict(key=["timestamp"], special=True), ) define_fake_model( diff --git a/tests/test_partitioning_time.py b/tests/test_partitioning_time.py index 0ab0daf..918635d 100644 --- a/tests/test_partitioning_time.py +++ b/tests/test_partitioning_time.py @@ -1,9 +1,11 @@ import datetime +from unittest.mock import patch import freezegun import pytest from dateutil.relativedelta import relativedelta +from django.core.management import call_command from django.db import connection, models, transaction from django.db.utils import IntegrityError @@ -535,6 +537,36 @@ def test_partitioning_time_multiple(kwargs, partition_names): assert partition_names == [par.name for par in table.partitions] +@pytest.mark.postgres_version(lt=110000) +@pytest.mark.parametrize( + "kwargs,partition_names", + [ + (dict(days=2), ["2018_dec_31", "2019_jan_02"]), + (dict(hours=2), ["2019_jan_01_00:00:00", "2019_jan_01_02:00:00"]), + (dict(weeks=2), ["2018_week_53", "2019_week_02"]), + (dict(months=2), ["2019_jan", "2019_mar"]), + (dict(years=2), ["2019", "2021"]), + ], +) +def test_partitioning_time_multiple_defer_attach(kwargs, partition_names): + model = define_fake_partitioned_model( + {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]} + ) + + schema_editor = connection.schema_editor() + schema_editor.create_partitioned_model(model) + + with freezegun.freeze_time("2019-1-1"): + manager = PostgresPartitioningManager( + [partition_by_current_time(model, **kwargs, count=2)] + ) + manager.plan(deferred_attach=True).apply() + + table = _get_partitioned_table(model) + assert len(table.partitions) == 2 + assert partition_names == [par.name for par in table.partitions] + + @pytest.mark.postgres_version(lt=110000) @pytest.mark.parametrize( "kwargs,timepoints", @@ -622,6 +654,46 @@ def test_partitioning_time_delete_ignore_manual(): assert len(table.partitions) == 1 +@pytest.mark.postgres_version(lt=110000) +@patch('psqlextra.management.commands.pgpartition.Command._partitioning_manager') +def test_partitioning_deferred_create(manager_mock): + """Tests that pgpartition honours deferred attach requests.""" + + model = define_fake_partitioned_model( + {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]} + ) + schema_editor = connection.schema_editor() + schema_editor.create_partitioned_model(model) + + manager = PostgresPartitioningManager( + [ + partition_by_current_time( + model, + years=1, + count=2, + max_age=relativedelta(years=1), + ) + ] + ) + manager_mock.return_value = manager + + with freezegun.freeze_time("2019-1-1"): + call_command("pgpartition", "--defer-attach", "--yes") + + table = _get_partitioned_table(model) + assert len(table.partitions) == 2 + assert table.partitions[0].name == "2019" + assert table.partitions[1].name == "2020" + + with freezegun.freeze_time("2020-01-01"): + call_command("pgpartition", "--defer-attach", "--yes") + + table = _get_partitioned_table(model) + assert len(table.partitions) == 2 + assert table.partitions[0].name == "2020" + assert table.partitions[1].name == "2021" + + def test_partitioning_time_no_size(): """Tests whether an error is raised when size for the partitions is specified.""" diff --git a/tests/test_schema_editor_partitioning.py b/tests/test_schema_editor_partitioning.py index d3602c0..3c07acf 100644 --- a/tests/test_schema_editor_partitioning.py +++ b/tests/test_schema_editor_partitioning.py @@ -411,3 +411,215 @@ def test_schema_editor_create_partitioned_custom_composite_primary_key( assert primary_key_constraint assert primary_key_constraint["columns"] == ["name", "timestamp"] + + +@pytest.mark.postgres_version(lt=140000) +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "method,key,add_partition_func_name,kwargs", + [ + ( + PostgresPartitioningMethod.RANGE, + ["timestamp"], + "add_range_partition", + {"from_values": "2019-1-1", "to_values": "2019-2-1"}, + ), + ( + PostgresPartitioningMethod.RANGE, + ["id"], + "add_range_partition", + {"from_values": 1, "to_values": 10}, + ), + ( + PostgresPartitioningMethod.LIST, + ["name"], + "add_list_partition", + {"values": ["1"]}, + ), + ], +) +def test_schema_editor_detach_and_delete_concurrently( + method, key, add_partition_func_name, kwargs +): + """Ensures detaching and deleting partitions works sequentially.""" + + model = define_fake_partitioned_model( + {"name": models.TextField(), "timestamp": models.DateTimeField()}, + {"method": method, "key": key}, + ) + + schema_editor = PostgresSchemaEditor(connection) + schema_editor.create_partitioned_model(model) + + getattr(schema_editor, add_partition_func_name)( + model, name="mypartition", comment="test", **kwargs, + ) + + table = db_introspection.get_partitioned_table(model._meta.db_table) + assert len(table.partitions) == 1 + assert table.partitions[0].name == "mypartition" + assert ( + table.partitions[0].full_name == f"{model._meta.db_table}_mypartition" + ) + assert table.partitions[0].comment == "test" + assert ( + f"{model._meta.db_table}_mypartition" in db_introspection.table_names() + ) + + schema_editor.detach_partition_concurrently(model, "mypartition") + schema_editor.delete_partition(model, "mypartition") + + table = db_introspection.get_partitioned_table(model._meta.db_table) + assert len(table.partitions) == 0 + assert ( + f"{model._meta.db_table}_mypartition" + not in db_introspection.table_names() + ) + + +@pytest.mark.postgres_version(lt=140000) +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "method,key,add_partition_func_name,kwargs", + [ + ( + PostgresPartitioningMethod.RANGE, + ["timestamp"], + "add_range_partition", + {"from_values": "2019-1-1", "to_values": "2019-2-1"}, + ), + ( + PostgresPartitioningMethod.RANGE, + ["id"], + "add_range_partition", + {"from_values": 1, "to_values": 10}, + ), + ( + PostgresPartitioningMethod.LIST, + ["name"], + "add_list_partition", + {"values": ["1"]}, + ), + ], +) +def test_schema_editor_detach_concurrently( + method, key, add_partition_func_name, kwargs +): + """Ensures detaching partitions leaves the detached table behind.""" + + model = define_fake_partitioned_model( + {"name": models.TextField(), "timestamp": models.DateTimeField()}, + {"method": method, "key": key}, + ) + + schema_editor = PostgresSchemaEditor(connection) + schema_editor.create_partitioned_model(model) + + getattr(schema_editor, add_partition_func_name)( + model, name="mypartition", comment="test", **kwargs, + ) + + table = db_introspection.get_partitioned_table(model._meta.db_table) + assert len(table.partitions) == 1 + assert table.partitions[0].name == "mypartition" + assert ( + table.partitions[0].full_name == f"{model._meta.db_table}_mypartition" + ) + assert table.partitions[0].comment == "test" + assert ( + f"{model._meta.db_table}_mypartition" in db_introspection.table_names() + ) + + schema_editor.detach_partition_concurrently(model, "mypartition") + + table = db_introspection.get_partitioned_table(model._meta.db_table) + assert len(table.partitions) == 0 + assert ( + f"{model._meta.db_table}_mypartition" in db_introspection.table_names() + ) + + +@pytest.mark.postgres_version(lt=140000) +@pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + "method,key,add_partition_func_name,kwargs", + [ + ( + PostgresPartitioningMethod.RANGE, + ["timestamp"], + "add_range_partition_deferred", + {"from_values": "2019-1-1", "to_values": "2019-2-1"}, + ), + ( + PostgresPartitioningMethod.RANGE, + ["id"], + "add_range_partition_deferred", + {"from_values": 1, "to_values": 10}, + ), + ], +) +def test_schema_editor_create_and_attach_and_then_detach_concurrently( + method, key, add_partition_func_name, kwargs +): + """Ensures deferred partitions can be detached once attached.""" + + model = define_fake_partitioned_model( + {"name": models.TextField(), "timestamp": models.DateTimeField()}, + {"method": method, "key": key}, + ) + + schema_editor = PostgresSchemaEditor(connection) + schema_editor.create_partitioned_model(model) + + getattr(schema_editor, add_partition_func_name)( + model, name="mypartition", comment="test", **kwargs, + ) + + table = db_introspection.get_partitioned_table(model._meta.db_table) + assert len(table.partitions) == 1 + assert table.partitions[0].name == "mypartition" + assert ( + table.partitions[0].full_name == f"{model._meta.db_table}_mypartition" + ) + assert table.partitions[0].comment == "test" + assert ( + f"{model._meta.db_table}_mypartition" in db_introspection.table_names() + ) + + +@pytest.mark.postgres_version(lt=110000) +@pytest.mark.django_db(transaction=True) +def test_schema_editor_detach_concurrently_within_context(): + """`detach_partition_concurrently` must succeed in schema editor contexts.""" + + model = define_fake_partitioned_model( + {"timestamp": models.DateTimeField()}, + {"method": PostgresPartitioningMethod.RANGE, "key": ["timestamp"]}, + ) + + schema_editor = PostgresSchemaEditor(connection) + + with schema_editor: + schema_editor.create_partitioned_model(model) + schema_editor.add_range_partition( + model, + name="mypartition", + from_values="2019-01-01", + to_values="2019-02-01", + ) + schema_editor.detach_partition_concurrently(model, "mypartition") + + table = db_introspection.get_partitioned_table(model._meta.db_table) + assert len(table.partitions) == 0 + assert ( + f"{model._meta.db_table}_mypartition" in db_introspection.table_names() + ) + + with schema_editor: + schema_editor.delete_partition(model, "mypartition") + schema_editor.delete_partitioned_model(model) + + assert ( + f"{model._meta.db_table}_mypartition" + not in db_introspection.table_names() + ) diff --git a/tests/test_schema_editor_view.py b/tests/test_schema_editor_view.py index ff20ef6..6bdead5 100644 --- a/tests/test_schema_editor_view.py +++ b/tests/test_schema_editor_view.py @@ -9,6 +9,7 @@ from .fake_model import ( define_fake_materialized_view_model, define_fake_view_model, + delete_fake_model, get_fake_model, ) @@ -174,3 +175,39 @@ def test_schema_editor_replace_materialized_view(): # replacing a materialized view involves re-creating it constraints_after = db_introspection.get_constraints(model._meta.db_table) assert constraints_after == constraints_before + + +@pytest.mark.django_db(transaction=True) +def test_schema_editor_refresh_materialized_view_concurrently_inside_context(): + """Refreshing concurrently within the schema editor context should work.""" + + underlying_model = get_fake_model({"name": models.TextField(unique=True)}) + + model = define_fake_materialized_view_model( + {"name": models.TextField()}, + {"query": underlying_model.objects.all()}, + ) + + underlying_model.objects.create(name="alpha") + underlying_model.objects.create(name="beta") + + schema_editor = PostgresSchemaEditor(connection) + schema_editor.create_materialized_view_model(model) + + # Postgres requires a unique index before allowing CONCURRENTLY refreshes. + with schema_editor: + schema_editor.execute( + "CREATE UNIQUE INDEX mv_refresh_unique_name ON %s (name)" + % schema_editor.quote_name(model._meta.db_table) + ) + + underlying_model.objects.create(name="gamma") + + with schema_editor: + schema_editor.refresh_materialized_view_model(model, concurrently=True) + + rows = set(model.objects.values_list("name", flat=True)) + assert rows == {"alpha", "beta", "gamma"} + + schema_editor.delete_materialized_view_model(model) + delete_fake_model(underlying_model) diff --git a/tests/test_strategy.py b/tests/test_strategy.py new file mode 100644 index 0000000..c332e21 --- /dev/null +++ b/tests/test_strategy.py @@ -0,0 +1,34 @@ +from typing import Generator + +from psqlextra.partitioning import ( + PostgresPartitioningStrategy, + PostgresPartition, + PostgresTimePartition, + PostgresRangePartition, +) +from psqlextra.partitioning.delete_on_condition_strategy import ( + PostgresDeleteOnConditionPartitioningStrategy, +) + + +class TestStrategy(PostgresPartitioningStrategy): + def to_create(self,) -> Generator[PostgresRangePartition, None, None]: + """Generates a list of partitions to be created.""" + + def to_delete(self,) -> Generator[PostgresRangePartition, None, None]: + """Generates a list of partitions to be deleted.""" + for i in range(0, 100, 10): + yield PostgresRangePartition(from_values=i, to_values=i + 9) + + +def test_delete_on_condition(): + def test_condition(partition: PostgresRangePartition): + return partition.to_values < 10 + + strategy = PostgresDeleteOnConditionPartitioningStrategy( + delegate=TestStrategy(), delete_condition=test_condition + ) + partitions_to_delete = list(filter(lambda x: x.to_be_deleted, list(strategy.to_delete()))) + assert len(partitions_to_delete) == 1 + assert partitions_to_delete[0].from_values == 0 + assert partitions_to_delete[0].to_values == 9 diff --git a/tox.ini b/tox.ini index a5a33ed..bac659e 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,10 @@ envlist = {py38,py39,py310,py311}-dj{41}-psycopg{28,29} {py310,py311,py312,py313}-dj{42,50,51,52}-psycopg{28,29,31,32} +; evaluation, flight_tracks +;envlist = py37-dj{21}, py39-dj{32} + + [testenv] deps = dj20: Django~=2.0.0