From 0a518111da12fd947163d1a820b12eb44de8a5b9 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Wed, 15 Feb 2023 17:34:30 -0800 Subject: [PATCH 01/38] Allow Django 4.2 (#227) * Allow Django 4.2 * allow Django 4.2 --- azure-pipelines.yml | 20 ++++++++++++++++++++ setup.py | 3 ++- tox.ini | 8 +++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3436990c..386ea5e7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,16 @@ jobs: strategy: matrix: + Python3.10 - Django 4.2: + python.version: '3.10' + tox.env: 'py310-django42' + Python 3.9 - Django 4.2: + python.version: '3.9' + tox.env: 'py39-django42' + Python 3.8 - Django 4.2: + python.version: '3.8' + tox.env: 'py38-django42' + Python3.10 - Django 4.1: python.version: '3.10' tox.env: 'py310-django41' @@ -111,6 +121,16 @@ jobs: strategy: matrix: + Python3.10 - Django 4.2: + python.version: '3.10' + tox.env: 'py310-django42' + Python 3.9 - Django 4.2: + python.version: '3.9' + tox.env: 'py39-django42' + Python 3.8 - Django 4.2: + python.version: '3.8' + tox.env: 'py38-django42' + Python3.10 - Django 4.1: python.version: '3.10' tox.env: 'py310-django41' diff --git a/setup.py b/setup.py index eba21613..f632e363 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ 'Framework :: Django :: 3.2', 'Framework :: Django :: 4.0', 'Framework :: Django :: 4.1', + 'Framework :: Django :: 4.2', ] this_directory = path.abspath(path.dirname(__file__)) @@ -39,7 +40,7 @@ license='BSD', packages=find_packages(), install_requires=[ - 'django>=2.2,<4.2', + 'django>=2.2,<4.3', 'pyodbc>=3.0', 'pytz', ], diff --git a/tox.ini b/tox.ini index 12d89b3c..91db1c4a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,10 @@ [tox] envlist = {py36,py37,py38,py39}-django32, - {py38, py39, py310}-django40 - {py38, py39, py310}-django41 - + {py38, py39, py310}-django40, + {py38, py39, py310}-django41, + {py38, py39, py310}-django42 + [testenv] allowlist_externals = bash @@ -19,3 +20,4 @@ deps = django32: django==3.2.* django40: django>=4.0a1,<4.1 django41: django>=4.1a1,<4.2 + django42: django>=4.2a1,<4.3 \ No newline at end of file From 81018de54c7669ba64d2ddac9927f3476da9d6db Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Tue, 21 Feb 2023 12:22:00 -0800 Subject: [PATCH 02/38] Fix errors with raising FullResultSet exception and with alter_column_type_sql() and collate_sql() functions (#229) * fix error with raising fullresultset * add django4.2 condition * fix alter_column_type_sql and collate_sql to take 2 additional arguments * delete argument 'old_rel_collation' * fix arguments names --- mssql/compiler.py | 16 ++++++++++++++-- mssql/schema.py | 37 ++++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/mssql/compiler.py b/mssql/compiler.py index e71aa14f..91ff9bc1 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -15,6 +15,8 @@ from django.db.utils import NotSupportedError if django.VERSION >= (3, 1): from django.db.models.fields.json import compile_json_path, KeyTransform as json_KeyTransform +if django.VERSION >= (4, 2): + from django.core.exceptions import FullResultSet def _as_sql_agv(self, compiler, connection): return self.as_sql(compiler, connection, template='%(function)s(CONVERT(float, %(field)s))') @@ -233,8 +235,18 @@ def as_sql(self, with_limits=True, with_col_aliases=False): # This must come after 'select', 'ordering', and 'distinct' -- see # docstring of get_from_clause() for details. from_, f_params = self.get_from_clause() - where, w_params = self.compile(self.where) if self.where is not None else ("", []) - having, h_params = self.compile(self.having) if self.having is not None else ("", []) + if django.VERSION >= (4, 2): + try: + where, w_params = self.compile(self.where) if self.where is not None else ("", []) + except FullResultSet: + where, w_params = "", [] + try: + having, h_params = self.compile(self.having) if self.having is not None else ("", []) + except FullResultSet: + having, h_params = "", [] + else: + where, w_params = self.compile(self.where) if self.where is not None else ("", []) + having, h_params = self.compile(self.having) if self.having is not None else ("", []) params = [] result = ['SELECT'] diff --git a/mssql/schema.py b/mssql/schema.py index f977fdc0..00ee5403 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -161,9 +161,14 @@ def _alter_column_null_sql(self, model, old_field, new_field): [], ) - def _alter_column_type_sql(self, model, old_field, new_field, new_type): - new_type = self._set_field_new_type_null_status(old_field, new_type) - return super()._alter_column_type_sql(model, old_field, new_field, new_type) + if django_version >= (4, 2): + def _alter_column_type_sql(self, model, old_field, new_field, new_type, old_collation, new_collation): + new_type = self._set_field_new_type_null_status(old_field, new_type) + return super()._alter_column_type_sql(model, old_field, new_field, new_type, old_collation, new_collation) + else: + def _alter_column_type_sql(self, model, old_field, new_field, new_type): + new_type = self._set_field_new_type_null_status(old_field, new_type) + return super()._alter_column_type_sql(model, old_field, new_field, new_type) def alter_unique_together(self, model, old_unique_together, new_unique_together): """ @@ -443,7 +448,12 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, post_actions = [] # Type change? if old_type != new_type: - fragment, other_actions = self._alter_column_type_sql(model, old_field, new_field, new_type) + if django_version >= (4, 2): + fragment, other_actions = self._alter_column_type_sql( + model, old_field, new_field, new_type, old_collation=None, new_collation=None + ) + else: + fragment, other_actions = self._alter_column_type_sql(model, old_field, new_field, new_type) actions.append(fragment) post_actions.extend(other_actions) # Drop unique constraint, SQL Server requires explicit deletion @@ -683,9 +693,14 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, for old_rel, new_rel in rels_to_update: rel_db_params = new_rel.field.db_parameters(connection=self.connection) rel_type = rel_db_params['type'] - fragment, other_actions = self._alter_column_type_sql( - new_rel.related_model, old_rel.field, new_rel.field, rel_type - ) + if django_version >= (4, 2): + fragment, other_actions = self._alter_column_type_sql( + new_rel.related_model, old_rel.field, new_rel.field, rel_type, old_collation=None, new_collation=None + ) + else: + fragment, other_actions = self._alter_column_type_sql( + new_rel.related_model, old_rel.field, new_rel.field, rel_type + ) # Drop related_model indexes, so it can be altered index_names = self._db_table_constraint_names(old_rel.related_model._meta.db_table, index=True) for index_name in index_names: @@ -1262,8 +1277,12 @@ def add_constraint(self, model, constraint): (constraint.condition.connector, constraint.name)) super().add_constraint(model, constraint) - def _collate_sql(self, collation): - return ' COLLATE ' + collation + if django_version >= (4, 2): + def _collate_sql(self, collation, old_collation=None, table_name=None): + return ' COLLATE ' + collation if collation else "" + else: + def _collate_sql(self, collation): + return ' COLLATE ' + collation def _create_index_name(self, table_name, column_names, suffix=""): index_name = super()._create_index_name(table_name, column_names, suffix) From 06e07885ac5ae9a0b6ac9333ad7a8d37ed302360 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Fri, 3 Mar 2023 10:01:04 -0800 Subject: [PATCH 03/38] fix last_executed_query() to properly replace placeholders with params (#234) --- mssql/operations.py | 8 +++++++- testapp/settings.py | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mssql/operations.py b/mssql/operations.py index 9222ea5e..fcc0608e 100644 --- a/mssql/operations.py +++ b/mssql/operations.py @@ -418,7 +418,13 @@ def last_executed_query(self, cursor, sql, params): exists for database backends to provide a better implementation according to their own quoting schemes. """ - return super().last_executed_query(cursor, cursor.last_sql, cursor.last_params) + if params: + if isinstance(params, list): + params = tuple(params) + return sql % params + # Just return sql when there are no parameters. + else: + return sql def savepoint_create_sql(self, sid): """ diff --git a/testapp/settings.py b/testapp/settings.py index 767f0f3d..311e9175 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -152,7 +152,6 @@ 'schema.tests.SchemaTests.test_unique_and_reverse_m2m', 'schema.tests.SchemaTests.test_unique_no_unnecessary_fk_drops', 'select_for_update.tests.SelectForUpdateTests.test_for_update_after_from', - 'backends.tests.LastExecutedQueryTest.test_last_executed_query', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_exact_lookup', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_greaterthan_lookup', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_lessthan_lookup', From 68751515146721466fd73b25ec9239af1ac27aca Mon Sep 17 00:00:00 2001 From: mShan0 Date: Fri, 10 Mar 2023 11:12:56 -0800 Subject: [PATCH 04/38] disable allows_group_by_select_index --- mssql/features.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mssql/features.py b/mssql/features.py index a60a9283..faaa0bbb 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -6,6 +6,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): + allows_group_by_select_index = False allow_sliced_subqueries_with_in = False can_introspect_autofield = True can_introspect_json_field = False @@ -71,4 +72,4 @@ def introspected_field_types(self): return { **super().introspected_field_types, "DurationField": "BigIntegerField", - } \ No newline at end of file + } From 9ac7c2f0c9bee278016d3973d3dc50b072397100 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Fri, 10 Mar 2023 11:33:37 -0800 Subject: [PATCH 05/38] unskip old tests --- testapp/settings.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/testapp/settings.py b/testapp/settings.py index 311e9175..ee5ee11d 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -99,11 +99,8 @@ TEST_RUNNER = "testapp.runners.ExcludedTestSuiteRunner" EXCLUDED_TESTS = [ - 'aggregation.tests.AggregateTestCase.test_expression_on_aggregation', - 'aggregation_regress.tests.AggregationTests.test_annotated_conditional_aggregate', 'aggregation_regress.tests.AggregationTests.test_annotation_with_value', 'aggregation.tests.AggregateTestCase.test_distinct_on_aggregate', - 'annotations.tests.NonAggregateAnnotationTestCase.test_annotate_exists', 'custom_lookups.tests.BilateralTransformTests.test_transform_order_by', 'expressions.tests.BasicExpressionsTests.test_filtering_on_annotate_that_uses_q', 'expressions.tests.BasicExpressionsTests.test_order_by_exists', @@ -165,17 +162,13 @@ 'backends.tests.BackendTestCase.test_queries', 'introspection.tests.IntrospectionTests.test_smallautofield', 'schema.tests.SchemaTests.test_inline_fk', - 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_exists', - 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_values_collision', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func_with_timezone', 'expressions.tests.FTimeDeltaTests.test_date_subquery_subtraction', 'expressions.tests.FTimeDeltaTests.test_datetime_subquery_subtraction', 'expressions.tests.FTimeDeltaTests.test_time_subquery_subtraction', 'migrations.test_operations.OperationTests.test_alter_field_reloads_state_on_fk_with_to_field_target_type_change', 'schema.tests.SchemaTests.test_alter_smallint_pk_to_smallautofield_pk', - 'annotations.tests.NonAggregateAnnotationTestCase.test_combined_expression_annotation_with_aggregation', - 'db_functions.comparison.test_cast.CastTests.test_cast_to_integer', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_func', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_iso_weekday_func', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func', @@ -271,7 +264,6 @@ # Django 4.1 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_on_exists', 'aggregation.tests.AggregateTestCase.test_aggregation_exists_multivalued_outeref', - 'annotations.tests.NonAggregateAnnotationTestCase.test_full_expression_annotation_with_aggregation', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_lookup_name_sql_injection', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_lookup_name_sql_injection', 'schema.tests.SchemaTests.test_autofield_to_o2o', From 9816cdfe212c853a922c506dc938b05f3b2ac1b4 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Mon, 13 Mar 2023 11:07:55 -0700 Subject: [PATCH 06/38] unskip some tests --- testapp/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testapp/settings.py b/testapp/settings.py index ee5ee11d..d87fc904 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -101,6 +101,7 @@ EXCLUDED_TESTS = [ 'aggregation_regress.tests.AggregationTests.test_annotation_with_value', 'aggregation.tests.AggregateTestCase.test_distinct_on_aggregate', + 'annotations.tests.NonAggregateAnnotationTestCase.test_annotate_exists', 'custom_lookups.tests.BilateralTransformTests.test_transform_order_by', 'expressions.tests.BasicExpressionsTests.test_filtering_on_annotate_that_uses_q', 'expressions.tests.BasicExpressionsTests.test_order_by_exists', @@ -162,6 +163,8 @@ 'backends.tests.BackendTestCase.test_queries', 'introspection.tests.IntrospectionTests.test_smallautofield', 'schema.tests.SchemaTests.test_inline_fk', + 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_exists', + 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_values_collision', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func_with_timezone', 'expressions.tests.FTimeDeltaTests.test_date_subquery_subtraction', 'expressions.tests.FTimeDeltaTests.test_datetime_subquery_subtraction', From 8e5c9cdac6918df92638d4f1bcfec62cd5cfb090 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Mon, 13 Mar 2023 12:16:39 -0700 Subject: [PATCH 07/38] skip more tests --- testapp/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testapp/settings.py b/testapp/settings.py index d87fc904..3fb637bb 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -267,6 +267,7 @@ # Django 4.1 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_on_exists', 'aggregation.tests.AggregateTestCase.test_aggregation_exists_multivalued_outeref', + 'annotations.tests.NonAggregateAnnotationTestCase.test_full_expression_annotation_with_aggregation', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_lookup_name_sql_injection', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_lookup_name_sql_injection', 'schema.tests.SchemaTests.test_autofield_to_o2o', From 303859c26cb40ebb116cd136a4a8c196d4f539a3 Mon Sep 17 00:00:00 2001 From: mShan0 <96149598+mShan0@users.noreply.github.com> Date: Wed, 15 Mar 2023 13:54:35 -0700 Subject: [PATCH 08/38] Use latest Django 4.2 beta for tox (#238) --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 91db1c4a..8737f9c0 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = {py38, py39, py310}-django40, {py38, py39, py310}-django41, {py38, py39, py310}-django42 - + [testenv] allowlist_externals = bash @@ -20,4 +20,4 @@ deps = django32: django==3.2.* django40: django>=4.0a1,<4.1 django41: django>=4.1a1,<4.2 - django42: django>=4.2a1,<4.3 \ No newline at end of file + django42: django>=4.2b1,<4.3 From e196f5a02f1867fb51b7789acbb9a157df8ebfc0 Mon Sep 17 00:00:00 2001 From: mShan0 <96149598+mShan0@users.noreply.github.com> Date: Mon, 20 Mar 2023 15:51:30 -0700 Subject: [PATCH 09/38] use 4.2 rc1 branch (#240) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8737f9c0..3376ddca 100644 --- a/tox.ini +++ b/tox.ini @@ -20,4 +20,4 @@ deps = django32: django==3.2.* django40: django>=4.0a1,<4.1 django41: django>=4.1a1,<4.2 - django42: django>=4.2b1,<4.3 + django42: django>=4.2rc1,<4.3 From 5a630ba52ccc26947129cbeee398b7a7c8aae7cc Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Thu, 23 Mar 2023 16:45:56 -0700 Subject: [PATCH 10/38] allow partial support for filtering against window functions (#239) --- mssql/compiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mssql/compiler.py b/mssql/compiler.py index 91ff9bc1..2dfe1b6c 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -230,6 +230,9 @@ def as_sql(self, with_limits=True, with_col_aliases=False): if not getattr(features, 'supports_select_{}'.format(combinator)): raise NotSupportedError('{} is not supported on this database backend.'.format(combinator)) result, params = self.get_combinator_sql(combinator, self.query.combinator_all) + elif django.VERSION >= (4, 2) and self.qualify: + result, params = self.get_qualify_sql() + order_by = None else: distinct_fields, distinct_params = self.get_distinct() # This must come after 'select', 'ordering', and 'distinct' -- see From adc1409feb09f63bdf1e2b5a5e17c9c2ff9b7403 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:10:08 -0700 Subject: [PATCH 11/38] add subsecond support to Now() (#242) --- mssql/functions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mssql/functions.py b/mssql/functions.py index ddf34ca7..dd7a1d3d 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -10,6 +10,7 @@ from django.db.models.expressions import Case, Exists, Expression, OrderBy, When, Window from django.db.models.fields import BinaryField, Field from django.db.models.functions import Cast, NthValue, MD5, SHA1, SHA224, SHA256, SHA384, SHA512 +from django.db.models.functions.datetime import Now from django.db.models.functions.math import ATan2, Ln, Log, Mod, Round, Degrees, Radians, Power from django.db.models.functions.text import Replace from django.db.models.lookups import In, Lookup @@ -123,6 +124,10 @@ def sqlserver_exists(self, compiler, connection, template=None, **extra_context) sql = 'CASE WHEN {} THEN 1 ELSE 0 END'.format(sql) return sql, params +def sqlserver_now(self, compiler, connection, **extra_context): + return self.as_sql( + compiler, connection, template="SYSDATETIME()", **extra_context + ) def sqlserver_lookup(self, compiler, connection): # MSSQL doesn't allow EXISTS() to be compared to another expression @@ -456,6 +461,7 @@ def sqlserver_sha512(self, compiler, connection, **extra_context): Round.as_microsoft = sqlserver_round Window.as_microsoft = sqlserver_window Replace.as_microsoft = sqlserver_replace +Now.as_microsoft = sqlserver_now MD5.as_microsoft = sqlserver_md5 SHA1.as_microsoft = sqlserver_sha1 SHA224.as_microsoft = sqlserver_sha224 From 0c4d052555d958ef17ee8278be40a74853acfb2c Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Tue, 4 Apr 2023 10:15:53 -0700 Subject: [PATCH 12/38] assign value to display_size (#244) --- mssql/introspection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index c5645f30..f23af078 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -107,7 +107,7 @@ def get_table_description(self, cursor, table_name, identity_check=True): """ # map pyodbc's cursor.columns to db-api cursor description - columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)] + columns = [[c[3], c[4], c[6], c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)] if not columns: raise DatabaseError(f"Table {table_name} does not exist.") From f9a1b43dcb61e9874b754be6ccc78260536addb6 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Tue, 4 Apr 2023 10:27:19 -0700 Subject: [PATCH 13/38] add latest django 4.2 branch to ci --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3376ddca..7d2794ef 100644 --- a/tox.ini +++ b/tox.ini @@ -20,4 +20,4 @@ deps = django32: django==3.2.* django40: django>=4.0a1,<4.1 django41: django>=4.1a1,<4.2 - django42: django>=4.2rc1,<4.3 + django42: django>=4.2,<4.3 From c7445a89ec206491d231cb99d9b94c2404af34b8 Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Mon, 17 Apr 2023 10:59:33 -0700 Subject: [PATCH 14/38] allow comments on columns and tables --- mssql/features.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mssql/features.py b/mssql/features.py index faaa0bbb..a27e90b7 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -33,6 +33,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): requires_literal_defaults = True requires_sqlparse_for_splitting = False supports_boolean_expr_in_select_clause = False + supports_comments = True supports_covering_indexes = True supports_deferrable_unique_constraints = False supports_expression_indexes = False From 4a37e3278c53724470b2423d129bfae7bf14c855 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Mon, 24 Apr 2023 12:44:42 -0700 Subject: [PATCH 15/38] raise an error when batch_size is zero. (#259) --- mssql/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/functions.py b/mssql/functions.py index dd7a1d3d..4fdedc7d 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -292,7 +292,7 @@ def bulk_update_with_default(self, objs, fields, batch_size=None, default=0): SQL Server require that at least one of the result expressions in a CASE specification must be an expression other than the NULL constant. Patched with a default value 0. The user can also pass a custom default value for CASE statement. """ - if batch_size is not None and batch_size < 0: + if batch_size is not None and batch_size <= 0: raise ValueError('Batch size must be a positive integer.') if not fields: raise ValueError('Field names must be given to bulk_update().') From 3375f3aba1fa7d94e2468dcc6e1da1218300a371 Mon Sep 17 00:00:00 2001 From: mShan0 <96149598+mShan0@users.noreply.github.com> Date: Wed, 3 May 2023 15:32:02 -0700 Subject: [PATCH 16/38] replicate get or create test for mssql (#265) --- testapp/migrations/0024_publisher_book.py | 58 +++++++++++++++++++++++ testapp/models.py | 20 +++++++- testapp/settings.py | 3 ++ testapp/tests/test_getorcreate.py | 41 ++++++++++++++++ 4 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 testapp/migrations/0024_publisher_book.py create mode 100644 testapp/tests/test_getorcreate.py diff --git a/testapp/migrations/0024_publisher_book.py b/testapp/migrations/0024_publisher_book.py new file mode 100644 index 00000000..b555d0d0 --- /dev/null +++ b/testapp/migrations/0024_publisher_book.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2 on 2023-05-03 15:08 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("testapp", "0023_number"), + ] + + operations = [ + migrations.CreateModel( + name="Publisher", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=100)), + ], + ), + migrations.CreateModel( + name="Book", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=100)), + ("updated", models.DateTimeField(auto_now=True)), + ( + "authors", + models.ManyToManyField(related_name="books", to="testapp.author"), + ), + ( + "publisher", + models.ForeignKey( + db_column="publisher_id_column", + on_delete=django.db.models.deletion.CASCADE, + related_name="books", + to="testapp.publisher", + ), + ), + ], + ), + ] diff --git a/testapp/models.py b/testapp/models.py index fb5fdbcf..f92d10f2 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -9,7 +9,7 @@ from django.db.models import Q from django.utils import timezone -# We are using this Mixin to test casting of BigAuto and Auto fields +# We are using this Mixin to test casting of BigAuto and Auto fields class BigAutoFieldMixin(models.Model): id = models.BigAutoField(primary_key=True) @@ -229,4 +229,20 @@ class Number(models.Model): decimal_value = models.DecimalField(max_digits=20, decimal_places=17, null=True) def __str__(self): - return "%i, %.3f, %.17f" % (self.integer, self.float, self.decimal_value) \ No newline at end of file + return "%i, %.3f, %.17f" % (self.integer, self.float, self.decimal_value) + + +class Publisher(models.Model): + name = models.CharField(max_length=100) + + +class Book(models.Model): + name = models.CharField(max_length=100) + authors = models.ManyToManyField(Author, related_name="books") + publisher = models.ForeignKey( + Publisher, + models.CASCADE, + related_name="books", + db_column="publisher_id_column", + ) + updated = models.DateTimeField(auto_now=True) diff --git a/testapp/settings.py b/testapp/settings.py index 3fb637bb..9593692d 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -289,6 +289,9 @@ 'model_fields.test_jsonfield.TestQuerying.test_lookups_with_key_transform', 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_count', 'model_fields.test_jsonfield.TestQuerying.test_has_key_number', + + # Django 4.2 + 'get_or_create.tests.UpdateOrCreateTests.test_update_only_defaults_and_pre_save_fields_when_local_fields' ] REGEX_TESTS = [ diff --git a/testapp/tests/test_getorcreate.py b/testapp/tests/test_getorcreate.py new file mode 100644 index 00000000..1fe9baf0 --- /dev/null +++ b/testapp/tests/test_getorcreate.py @@ -0,0 +1,41 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the BSD license. +from unittest import skipUnless + +from django import VERSION +from django.test import TestCase +from django.db import connection +from django.test.utils import CaptureQueriesContext + +from ..models import Book, Publisher + +DJANGO42 = VERSION >= (4, 2) + +# Copied from Django test suite but modified to test our code +@skipUnless(DJANGO42, "Django 4.2 specific tests") +class UpdateOrCreateTests(TestCase): + + def test_update_only_defaults_and_pre_save_fields_when_local_fields(self): + publisher = Publisher.objects.create(name="Acme Publishing") + book = Book.objects.create(publisher=publisher, name="The Book of Ed & Fred") + + for defaults in [{"publisher": publisher}, {"publisher_id": publisher}]: + with self.subTest(defaults=defaults): + with CaptureQueriesContext(connection) as captured_queries: + book, created = Book.objects.update_or_create( + pk=book.pk, + defaults=defaults, + ) + self.assertIs(created, False) + update_sqls = [ + q["sql"] for q in captured_queries if "UPDATE" in q["sql"] + ] + self.assertEqual(len(update_sqls), 1) + update_sql = update_sqls[0] + self.assertIsNotNone(update_sql) + self.assertIn( + connection.ops.quote_name("publisher_id_column"), update_sql + ) + self.assertIn(connection.ops.quote_name("updated"), update_sql) + # Name should not be updated. + self.assertNotIn(connection.ops.quote_name("name"), update_sql) From 068c7fc201e48838845993caec91ab1edabfa926 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Tue, 23 May 2023 14:44:03 -0700 Subject: [PATCH 17/38] add table comment to `get_table_list` query --- mssql/introspection.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index f23af078..5e0feadd 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -4,10 +4,12 @@ from django.db import DatabaseError import pyodbc as Database +from collections import namedtuple + from django import VERSION -from django.db.backends.base.introspection import ( - BaseDatabaseIntrospection, FieldInfo, TableInfo, -) +from django.db.backends.base.introspection import BaseDatabaseIntrospection +from django.db.backends.base.introspection import FieldInfo +from django.db.backends.base.introspection import TableInfo as BaseTableInfo from django.db.models.indexes import Index from django.conf import settings @@ -15,6 +17,7 @@ SQL_BIGAUTOFIELD = -777444 SQL_TIMESTAMP_WITH_TIMEZONE = -155 +TableInfo = namedtuple("TableInfo", BaseTableInfo._fields + ("comment",)) def get_schema_name(): return getattr(settings, 'SCHEMA_TO_INSPECT', 'SCHEMA_NAME()') @@ -71,13 +74,28 @@ def get_table_list(self, cursor): """ Returns a list of table and view names in the current database. """ - sql = 'SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = %s' % ( + sql = """SELECT + TABLE_NAME, + TABLE_TYPE, + CAST(ep.value AS VARCHAR) AS COMMENT + FROM INFORMATION_SCHEMA.TABLES i + INNER JOIN sys.tables t ON t.name = i.TABLE_NAME + LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id + WHERE + ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id = 0) OR ep.value IS NULL) + AND + i.TABLE_SCHEMA = %s""" % ( get_schema_name()) cursor.execute(sql) types = {'BASE TABLE': 't', 'VIEW': 'v'} - return [TableInfo(row[0], types.get(row[1])) - for row in cursor.fetchall() - if row[0] not in self.ignored_tables] + if VERSION >= (4, 2): + return [TableInfo(row[0], types.get(row[1]), row[2]) + for row in cursor.fetchall() + if row[0] not in self.ignored_tables] + else: + return [BaseTableInfo(row[0], types.get(row[1])) + for row in cursor.fetchall() + if row[0] not in self.ignored_tables] def _is_auto_field(self, cursor, table_name, column_name): """ @@ -111,7 +129,7 @@ def get_table_description(self, cursor, table_name, identity_check=True): if not columns: raise DatabaseError(f"Table {table_name} does not exist.") - + items = [] for column in columns: if VERSION >= (3, 2): From e9383603fffa7915aafee4cf2e4511100cdc03cd Mon Sep 17 00:00:00 2001 From: mShan0 Date: Tue, 23 May 2023 15:57:20 -0700 Subject: [PATCH 18/38] add column comment to `get_table_description()` --- mssql/introspection.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index 5e0feadd..1e022f51 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -8,7 +8,7 @@ from django import VERSION from django.db.backends.base.introspection import BaseDatabaseIntrospection -from django.db.backends.base.introspection import FieldInfo +from django.db.backends.base.introspection import FieldInfo as BaseFieldInfo from django.db.backends.base.introspection import TableInfo as BaseTableInfo from django.db.models.indexes import Index from django.conf import settings @@ -17,6 +17,7 @@ SQL_BIGAUTOFIELD = -777444 SQL_TIMESTAMP_WITH_TIMEZONE = -155 +FieldInfo = namedtuple("FieldInfo", BaseFieldInfo._fields + ("comment",)) TableInfo = namedtuple("TableInfo", BaseTableInfo._fields + ("comment",)) def get_schema_name(): @@ -144,7 +145,16 @@ def get_table_description(self, cursor, table_name, identity_check=True): column.append(collation_name[0] if collation_name else '') else: column.append('') - + if VERSION >= (4, 2): + sql = """select CAST(ep.value AS VARCHAR) AS COMMENT + FROM sys.columns c + INNER JOIN sys.tables t ON c.object_id = t.object_id + INNER JOIN sys.extended_properties ep ON c.object_id=ep.major_id AND ep.minor_id = c.column_id + WHERE t.name = '%s' AND c.name = '%s' AND ep.name = 'MS_Description' + """ % (table_name, column[0]) + cursor.execute(sql) + comment = cursor.fetchone() + column.append(comment) if identity_check and self._is_auto_field(cursor, table_name, column[0]): if column[1] == Database.SQL_BIGINT: column[1] = SQL_BIGAUTOFIELD @@ -152,7 +162,10 @@ def get_table_description(self, cursor, table_name, identity_check=True): column[1] = SQL_AUTOFIELD if column[1] == Database.SQL_WVARCHAR and column[3] < 4000: column[1] = Database.SQL_WCHAR - items.append(FieldInfo(*column)) + if VERSION >= (4, 2): + items.append(FieldInfo(*column)) + else: + items.append(BaseFieldInfo(*column)) return items def get_sequences(self, cursor, table_name, table_fields=()): From 852d67daf340effa551242f1d3dac9192ab2da9d Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Thu, 25 May 2023 12:37:22 -0700 Subject: [PATCH 19/38] return column comment only for `get_table_description()` --- mssql/introspection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index 1e022f51..16381fdf 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -154,7 +154,7 @@ def get_table_description(self, cursor, table_name, identity_check=True): """ % (table_name, column[0]) cursor.execute(sql) comment = cursor.fetchone() - column.append(comment) + column.append(comment[0] if comment else '') if identity_check and self._is_auto_field(cursor, table_name, column[0]): if column[1] == Database.SQL_BIGINT: column[1] = SQL_BIGAUTOFIELD From 0a2664ad020a426f0ab18a79526194c778e7ab95 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Tue, 30 May 2023 15:03:39 -0700 Subject: [PATCH 20/38] Add skipped tests to Django 4.2 (#268) * skip django 4.2 failing tests * skip schema test * skip aggregate annotation pruning test --------- Co-authored-by: mShan0 --- testapp/settings.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/testapp/settings.py b/testapp/settings.py index 9593692d..a2429a8b 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -291,7 +291,17 @@ 'model_fields.test_jsonfield.TestQuerying.test_has_key_number', # Django 4.2 - 'get_or_create.tests.UpdateOrCreateTests.test_update_only_defaults_and_pre_save_fields_when_local_fields' + 'get_or_create.tests.UpdateOrCreateTests.test_update_only_defaults_and_pre_save_fields_when_local_fields', + 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_empty_condition', + 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_multiple_subquery_annotation', + 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_subquery_annotation', + "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_group_by_annotation_kept" + 'aggregation.tests.AggregateTestCase.test_group_by_nested_expression_with_params', + 'expressions.tests.BasicExpressionsTests.test_aggregate_subquery_annotation', + 'queries.test_qs_combinators.QuerySetSetOperationTests.test_union_order_with_null_first_last', + 'queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_select_related_and_order', + 'expressions_window.tests.WindowFunctionTests.test_limited_filter', + 'schema.tests.SchemaTests.test_remove_ignored_unique_constraint_not_create_fk_index', ] REGEX_TESTS = [ From a3e895bc8691e9d2557e202258a0fca2f84a7072 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Wed, 31 May 2023 09:55:40 -0700 Subject: [PATCH 21/38] syntax fix --- testapp/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testapp/settings.py b/testapp/settings.py index 61a18613..ecdafbde 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -292,7 +292,7 @@ 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_empty_condition', 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_multiple_subquery_annotation', 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_subquery_annotation', - "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_group_by_annotation_kept" + "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_group_by_annotation_kept", 'aggregation.tests.AggregateTestCase.test_group_by_nested_expression_with_params', 'expressions.tests.BasicExpressionsTests.test_aggregate_subquery_annotation', 'queries.test_qs_combinators.QuerySetSetOperationTests.test_union_order_with_null_first_last', From 19e15390a4922222ad311c68c16aed36a23ff304 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Wed, 31 May 2023 12:31:09 -0700 Subject: [PATCH 22/38] ci fix --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ffd360a6..7cb512dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -133,10 +133,10 @@ jobs: matrix: Python3.11 - Django 4.2: python.version: '3.11' - tox.env: 'py310-django42' + tox.env: 'py311-django42' Python3.10 - Django 4.2: python.version: '3.10' - tox.env: 'py310-django42' + tox.env: 'py310-django42' Python 3.9 - Django 4.2: python.version: '3.9' tox.env: 'py39-django42' From 2885672004ff875fe32e60df2bb8daa5eafd0ffe Mon Sep 17 00:00:00 2001 From: mShan0 <96149598+mShan0@users.noreply.github.com> Date: Wed, 31 May 2023 14:03:45 -0700 Subject: [PATCH 23/38] bump version to 1.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 82d5ee0e..78532e94 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( name='mssql-django', - version='1.2', + version='1.3', description='Django backend for Microsoft SQL Server', long_description=long_description, long_description_content_type='text/markdown', From eec621870e2bddb3189d8467d65102c0315bce4d Mon Sep 17 00:00:00 2001 From: Nasreddine Date: Tue, 20 Jun 2023 15:54:13 +0200 Subject: [PATCH 24/38] added schema support --- mssql/compiler.py | 136 ++++++++++++++++-- mssql/introspection.py | 39 +++-- mssql/schema.py | 219 ++++++++++++++++++++++------- testapp/migrations/0001_initial.py | 12 +- testapp/models.py | 10 +- testapp/settings.py | 1 + testapp/tests/test_queries.py | 2 +- 7 files changed, 339 insertions(+), 80 deletions(-) diff --git a/mssql/compiler.py b/mssql/compiler.py index 2dfe1b6c..20220ce7 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -18,6 +18,8 @@ if django.VERSION >= (4, 2): from django.core.exceptions import FullResultSet +from .introspection import get_table_name, get_schema_name + def _as_sql_agv(self, compiler, connection): return self.as_sql(compiler, connection, template='%(function)s(CONVERT(float, %(field)s))') @@ -196,7 +198,6 @@ def _cursor_iter(cursor, sentinel, col_count, itersize): compiler.cursor_iter = _cursor_iter - class SQLCompiler(compiler.SQLCompiler): def as_sql(self, with_limits=True, with_col_aliases=False): @@ -227,6 +228,7 @@ def as_sql(self, with_limits=True, with_col_aliases=False): do_offset_emulation = do_offset and not supports_offset_clause if combinator: + if not getattr(features, 'supports_select_{}'.format(combinator)): raise NotSupportedError('{} is not supported on this database backend.'.format(combinator)) result, params = self.get_combinator_sql(combinator, self.query.combinator_all) @@ -280,7 +282,8 @@ def as_sql(self, with_limits=True, with_col_aliases=False): if do_offset: meta = self.query.get_meta() qn = self.quote_name_unless_alias - offsetting_order_by = '%s.%s' % (qn(meta.db_table), qn(meta.pk.db_column or meta.pk.column)) + table = qn(get_table_name(self, meta.db_table, getattr(meta, "db_table_schema", False))) + offsetting_order_by = '%s.%s' % (table, qn(meta.pk.db_column or meta.pk.column)) if do_offset_emulation: if order_by: ordering = [] @@ -426,12 +429,53 @@ def as_sql(self, with_limits=True, with_col_aliases=False): ', '.join(sub_selects), ' '.join(result), ), tuple(sub_params + params) - return ' '.join(result), tuple(params) finally: # Finally do cleanup - get rid of the joins we created above. self.query.reset_refcounts(refcounts_before) - + def get_from_clause(self): + """ + Return a list of strings that are joined together to go after the + "FROM" part of the query, as well as a list any extra parameters that + need to be included. Subclasses, can override this to create a + from-clause via a "select". + + This should only be called after any SQL construction methods that + might change the tables that are needed. This means the select columns, + ordering, and distinct must be done first. + """ + result = [] + params = [] + for alias in tuple(self.query.alias_map): + if not self.query.alias_refcount[alias]: + continue + try: + from_clause = self.query.alias_map[alias] + except KeyError: + # Extra tables can end up in self.tables, but not in the + # alias_map if they aren't in a join. That's OK. We skip them. + continue + opts = self.query.get_meta() + settings_dict = self.connection.settings_dict + schema = getattr(opts, "db_table_schema", settings_dict.get('SCHEMA', False)) + clause_sql, clause_params = self.compile(from_clause) + if schema: + if 'JOIN' in clause_sql: + table_clause_sql = clause_sql.split('JOIN ')[1].split(' ON')[0] + table_clause_sql = f'[{schema}].{table_clause_sql}' + clause_sql = clause_sql.split('JOIN')[0] + 'JOIN ' + table_clause_sql + ' ON' + clause_sql.split('JOIN')[1].split('ON')[1] + else: + clause_sql = f'[{schema}].{clause_sql}' + result.append(clause_sql) + params.extend(clause_params) + for t in self.query.extra_tables: + alias, _ = self.query.table_alias(t) + # Only add the alias if it's not already present (the table_alias() + # call increments the refcount, so an alias refcount of one means + # this is the only reference). + if alias not in self.query.alias_map or self.query.alias_refcount[alias] == 1: + result.append(', %s' % self.quote_name_unless_alias(alias)) + return result, params def compile(self, node, *args, **kwargs): node = self._as_microsoft(node) return super().compile(node, *args, **kwargs) @@ -507,7 +551,7 @@ def fix_auto(self, sql, opts, fields, qn): columns = [f.column for f in fields] if auto_field_column in columns: id_insert_sql = [] - table = qn(opts.db_table) + table = qn(get_table_name(self, opts.db_table, getattr(opts, "db_table_schema", False))) sql_format = 'SET IDENTITY_INSERT %s ON; %s; SET IDENTITY_INSERT %s OFF' for q, p in sql: id_insert_sql.append((sql_format % (table, q, table), p)) @@ -544,7 +588,8 @@ def as_sql(self): # going to be column names (so we can avoid the extra overhead). qn = self.connection.ops.quote_name opts = self.query.get_meta() - result = ['INSERT INTO %s' % qn(opts.db_table)] + table = qn(get_table_name(self, opts.db_table, getattr(opts, "db_table_schema", False))) + result = ['INSERT INTO %s' % table] if self.query.fields: fields = self.query.fields @@ -574,7 +619,7 @@ def as_sql(self): # There isn't really a single statement to bulk multiple DEFAULT VALUES insertions, # so we have to use a workaround: # https://dba.stackexchange.com/questions/254771/insert-multiple-rows-into-a-table-with-only-an-identity-column - result = [self.bulk_insert_default_values_sql(qn(opts.db_table))] + result = [self.bulk_insert_default_values_sql(qn(table))] r_sql, self.returning_params = self.connection.ops.return_insert_columns(self.get_returned_fields()) if r_sql: result.append(r_sql) @@ -617,13 +662,82 @@ def as_sql(self): sql = '; '.join(['SET NOCOUNT OFF', sql]) return sql, params + def _as_sql(self, query): + opts = self.query.get_meta() + table = get_table_name(self, query.base_table, getattr(opts, "db_table_schema", False)) + delete = "DELETE FROM %s" % self.quote_name_unless_alias(table) + try: + where, params = self.compile(query.where) + except FullResultSet: + return delete, () + return f"{delete} WHERE {where}", tuple(params) + class SQLUpdateCompiler(compiler.SQLUpdateCompiler, SQLCompiler): def as_sql(self): - sql, params = super().as_sql() - if sql: - sql = '; '.join(['SET NOCOUNT OFF', sql]) - return sql, params + """ + Create the SQL for this query. Return the SQL string and list of + parameters. + """ + self.pre_sql_setup() + if not self.query.values: + return "", () + qn = self.quote_name_unless_alias + values, update_params = [], [] + for field, model, val in self.query.values: + if hasattr(val, "resolve_expression"): + val = val.resolve_expression( + self.query, allow_joins=False, for_save=True + ) + if val.contains_aggregate: + raise FieldError( + "Aggregate functions are not allowed in this query " + "(%s=%r)." % (field.name, val) + ) + if val.contains_over_clause: + raise FieldError( + "Window expressions are not allowed in this query " + "(%s=%r)." % (field.name, val) + ) + elif hasattr(val, "prepare_database_save"): + if field.remote_field: + val = val.prepare_database_save(field) + else: + raise TypeError( + "Tried to update field %s with a model instance, %r. " + "Use a value compatible with %s." + % (field, val, field.__class__.__name__) + ) + val = field.get_db_prep_save(val, connection=self.connection) + + # Getting the placeholder for the field. + if hasattr(field, "get_placeholder"): + placeholder = field.get_placeholder(val, self, self.connection) + else: + placeholder = "%s" + name = field.column + if hasattr(val, "as_sql"): + sql, params = self.compile(val) + values.append("%s = %s" % (qn(name), placeholder % sql)) + update_params.extend(params) + elif val is not None: + values.append("%s = %s" % (qn(name), placeholder)) + update_params.append(val) + else: + values.append("%s = NULL" % qn(name)) + opts = self.query.get_meta() + table = get_table_name(self, self.query.base_table, getattr(opts, "db_table_schema", False)) + result = [ + "UPDATE %s SET" % qn(table), + ", ".join(values), + ] + try: + where, params = self.compile(self.query.where) + except FullResultSet: + params = [] + else: + result.append("WHERE %s" % where) + return " ".join(result), tuple(update_params + params) class SQLAggregateCompiler(compiler.SQLAggregateCompiler, SQLCompiler): diff --git a/mssql/introspection.py b/mssql/introspection.py index f23af078..9b7e1e2d 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -14,11 +14,32 @@ SQL_AUTOFIELD = -777555 SQL_BIGAUTOFIELD = -777444 SQL_TIMESTAMP_WITH_TIMEZONE = -155 +from django.db import connection def get_schema_name(): - return getattr(settings, 'SCHEMA_TO_INSPECT', 'SCHEMA_NAME()') + # get default schema choosen by user in settings.py else SCHEMA_NAME() + settings_dict = connection.settings_dict + schema = settings_dict.get('SCHEMA', False) + return f"'{schema}'" if schema else 'SCHEMA_NAME()' +def get_table_name(object, table_name, custom_schema): + """ + get the name of the table on this format schema].[table_name + if + schema = custom schema defined in medels meta (db_table_schema) + else + schema choosen by user in settings.py + else + return the name of the table without schema (defalut one will be used) + """ + if custom_schema: + return f'{custom_schema}].[{table_name}' + settings_dict = object.connection.settings_dict + schema_name = settings_dict.get('SCHEMA', False) + if schema_name: + return f'{schema_name}].[{table_name}' + return table_name class DatabaseIntrospection(BaseDatabaseIntrospection): # Map type codes to Django Field types. @@ -71,13 +92,13 @@ def get_table_list(self, cursor): """ Returns a list of table and view names in the current database. """ - sql = 'SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = %s' % ( - get_schema_name()) + sql = 'SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = %s' % (get_schema_name()) cursor.execute(sql) types = {'BASE TABLE': 't', 'VIEW': 'v'} - return [TableInfo(row[0], types.get(row[1])) + list = [TableInfo(row[0], types.get(row[1])) for row in cursor.fetchall() if row[0] not in self.ignored_tables] + return list def _is_auto_field(self, cursor, table_name, column_name): """ @@ -191,7 +212,7 @@ def get_key_columns(self, cursor, table_name): key_columns.extend([tuple(row) for row in cursor.fetchall()]) return key_columns - def get_constraints(self, cursor, table_name): + def get_constraints(self, cursor, table_name, table_name_schema='SCHEMA_NAME()'): """ Retrieves any constraints or keys (unique, pk, fk, check, index) across one or more columns. @@ -250,12 +271,12 @@ def get_constraints(self, cursor, table_name): kc.table_name = fk.table_name AND kc.column_name = fk.column_name WHERE - kc.table_schema = {get_schema_name()} AND + kc.table_schema = {table_name_schema} AND kc.table_name = %s ORDER BY kc.constraint_name ASC, kc.ordinal_position ASC - """, [table_name]) + """ , [table_name]) for constraint, column, kind, ref_table, ref_column in cursor.fetchall(): # If we're the first column, make the record if constraint not in constraints: @@ -284,7 +305,7 @@ def get_constraints(self, cursor, table_name): kc.constraint_name = c.constraint_name WHERE c.constraint_type = 'CHECK' AND - kc.table_schema = {get_schema_name()} AND + kc.table_schema = {table_name_schema} AND kc.table_name = %s """, [table_name]) for constraint, column in cursor.fetchall(): @@ -325,7 +346,7 @@ def get_constraints(self, cursor, table_name): ic.object_id = c.object_id AND ic.column_id = c.column_id WHERE - t.schema_id = SCHEMA_ID({get_schema_name()}) AND + t.schema_id = SCHEMA_ID({table_name_schema}) AND t.name = %s ORDER BY i.index_id ASC, diff --git a/mssql/schema.py b/mssql/schema.py index 00ee5403..55441f8d 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -15,6 +15,7 @@ from django.db.backends.ddl_references import ( Columns, IndexName, + ForeignKeyName, Statement as DjStatement, Table, ) @@ -29,6 +30,8 @@ from django.db.models.sql import Query from django.db.backends.ddl_references import Expressions +from .introspection import get_table_name, get_schema_name + class Statement(DjStatement): def __hash__(self): @@ -111,9 +114,10 @@ def _alter_column_default_sql(self, model, old_field, new_field, drop=False): if drop: params = [] # SQL Server requires the name of the default constraint + db_table = model._meta.db_table result = self.execute( self._sql_select_default_constraint_name % { - "table": self.quote_value(model._meta.db_table), + "table": self.quote_value(db_table), "column": self.quote_value(new_field.column), }, has_result=True @@ -239,7 +243,7 @@ def _model_indexes_sql(self, model): output.append(index.create_sql(model, self)) return output - def _db_table_constraint_names(self, db_table, column_names=None, column_match_any=False, + def _db_table_constraint_names(self, model, column_names=None, column_match_any=False, unique=None, primary_key=None, index=None, foreign_key=None, check=None, type_=None, exclude=None, unique_constraint=None): """ @@ -250,13 +254,16 @@ def _db_table_constraint_names(self, db_table, column_names=None, column_match_a False: (default) only return constraints covering exactly `column_names` True : return any constraints which include at least 1 of `column_names` """ + db_table = model._meta.db_table + db_table_schema = getattr(model._meta, "db_table_schema", False) + db_table_schema = f"'{db_table_schema}'" if db_table_schema else get_schema_name() if column_names is not None: column_names = [ self.connection.introspection.identifier_converter(name) for name in column_names ] with self.connection.cursor() as cursor: - constraints = self.connection.introspection.get_constraints(cursor, db_table) + constraints = self.connection.introspection.get_constraints(cursor, db_table, db_table_schema) result = [] for name, infodict in constraints.items(): if column_names is None or column_names == infodict['columns'] or ( @@ -303,6 +310,9 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # the backend doesn't support altering a column to/from AutoField as # SQL Server cannot alter columns to add and remove IDENTITY properties + + db_table = model._meta.db_table + table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) old_is_auto = False new_is_auto = False for t in (AutoField, BigAutoField): @@ -327,7 +337,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, if strict and len(fk_names) != 1: raise ValueError("Found wrong number (%s) of foreign key constraints for %s.%s" % ( len(fk_names), - model._meta.db_table, + table, old_field.column, )) for fk_name in fk_names: @@ -401,7 +411,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, if strict and len(constraint_names) != 1: raise ValueError("Found wrong number (%s) of check constraints for %s.%s" % ( len(constraint_names), - model._meta.db_table, + table, old_field.column, )) for constraint_name in constraint_names: @@ -411,7 +421,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, sql_restore_index = '' # Drop any unique indexes which include the column to be renamed index_names = self._db_table_constraint_names( - db_table=model._meta.db_table, column_names=[old_field.column], column_match_any=True, + model=model, column_names=[old_field.column], column_match_any=True, index=True, unique=True, ) for index_name in index_names: @@ -423,24 +433,24 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, FROM sys.indexes AS i INNER JOIN sys.index_columns AS ic ON i.object_id = ic.object_id AND i.index_id = ic.index_id - WHERE i.object_id = OBJECT_ID('{model._meta.db_table}') + WHERE i.object_id = OBJECT_ID('{db_table}') and i.name = '{index_name}' """) result = cursor.fetchall() columns_to_recreate_index = ', '.join(['%s' % self.quote_name(column[0]) for column in result]) filter_definition = result[0][1] sql_restore_index += 'CREATE UNIQUE INDEX %s ON %s (%s) WHERE %s;' % ( - index_name, model._meta.db_table, columns_to_recreate_index, filter_definition) + index_name, '[' + table + ']', columns_to_recreate_index, filter_definition) self.execute(self._db_table_delete_constraint_sql( - self.sql_delete_index, model._meta.db_table, index_name)) - self.execute(self._rename_field_sql(model._meta.db_table, old_field, new_field, new_type)) + self.sql_delete_index, table, index_name)) + self.execute(self._rename_field_sql(table, old_field, new_field, new_type)) # Restore index(es) now the column has been renamed if sql_restore_index: self.execute(sql_restore_index.replace(f'[{old_field.column}]', f'[{new_field.column}]')) # Rename all references to the renamed column. for sql in self.deferred_sql: if isinstance(sql, DjStatement): - sql.rename_column_references(model._meta.db_table, old_field.column, new_field.column) + sql.rename_column_references(table, old_field.column, new_field.column) # Next, start accumulating actions to do actions = [] @@ -489,7 +499,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, indexes_dropped = self._delete_indexes(model, old_field, new_field) auto_index_names = [] for index_from_meta in model._meta.indexes: - auto_index_names.append(self._create_index_name(model._meta.db_table, index_from_meta.fields)) + auto_index_names.append(self._create_index_name(db_table, index_from_meta.fields)) if ( new_field.get_internal_type() not in ("JSONField", "TextField") and @@ -519,7 +529,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, for sql, params in actions: self.execute( self.sql_alter_column % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "changes": sql, }, params, @@ -528,7 +538,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # Update existing rows with default value self.execute( self.sql_update_with_default % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "column": self.quote_name(new_field.column), "default": "%s", }, @@ -539,7 +549,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, for sql, params in null_actions: self.execute( self.sql_alter_column % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "changes": sql, }, params, @@ -617,9 +627,9 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, if old_field.primary_key and new_field.primary_key: self.execute( self.sql_create_pk % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "name": self.quote_name( - self._create_index_name(model._meta.db_table, [new_field.column], suffix="_pk") + self._create_index_name(db_table, [new_field.column], suffix="_pk") ), "columns": self.quote_name(new_field.column), } @@ -680,9 +690,9 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # Make the new one self.execute( self.sql_create_pk % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "name": self.quote_name( - self._create_index_name(model._meta.db_table, [new_field.column], suffix="_pk") + self._create_index_name(db_table, [new_field.column], suffix="_pk") ), "columns": self.quote_name(new_field.column), } @@ -702,7 +712,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, new_rel.related_model, old_rel.field, new_rel.field, rel_type ) # Drop related_model indexes, so it can be altered - index_names = self._db_table_constraint_names(old_rel.related_model._meta.db_table, index=True) + index_names = self._db_table_constraint_names(old_rel.related_model, index=True) for index_name in index_names: self.execute(self._db_table_delete_constraint_sql( self.sql_delete_index, old_rel.related_model._meta.db_table, index_name)) @@ -754,9 +764,9 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, ): self.execute( self.sql_create_check % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "name": self.quote_name( - self._create_index_name(model._meta.db_table, [new_field.column], suffix="_check") + self._create_index_name(db_table, [new_field.column], suffix="_check") ), "column": self.quote_name(new_field.column), "check": new_db_params['check'], @@ -767,7 +777,7 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, if needs_database_default: changes_sql, params = self._alter_column_default_sql(model, old_field, new_field, drop=True) sql = self.sql_alter_column % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "changes": changes_sql, } self.execute(sql, params) @@ -821,12 +831,12 @@ def _delete_unique_constraints(self, model, old_field, new_field, strict=False): def _delete_unique_constraint_for_columns(self, model, columns, strict=False, **constraint_names_kwargs): constraint_names_unique = self._db_table_constraint_names( - model._meta.db_table, columns, unique=True, unique_constraint=True, **constraint_names_kwargs) + model, columns, unique=True, unique_constraint=True, **constraint_names_kwargs) constraint_names_primary = self._db_table_constraint_names( - model._meta.db_table, columns, unique=True, primary_key=True, **constraint_names_kwargs) + model, columns, unique=True, primary_key=True, **constraint_names_kwargs) constraint_names_normal = constraint_names_unique + constraint_names_primary constraint_names_index = self._db_table_constraint_names( - model._meta.db_table, columns, unique=True, unique_constraint=False, primary_key=False, + model, columns, unique=True, unique_constraint=False, primary_key=False, **constraint_names_kwargs) constraint_names = constraint_names_normal + constraint_names_index if django_version >= (4, 1): @@ -871,6 +881,9 @@ def add_field(self, model, field): Create a field on a model. Usually involves adding a column, but may involve adding a table instead (for M2M fields). """ + + db_table = model._meta.db_table + table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) # Special-case implicit M2M tables if field.many_to_many and field.remote_field.through._meta.auto_created: return self.create_model(field.remote_field.through) @@ -896,7 +909,7 @@ def add_field(self, model, field): definition += " CHECK (%s)" % db_params['check'] # Build the SQL and run it sql = self.sql_create_column % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "column": self.quote_name(field.column), "definition": definition, } @@ -906,7 +919,7 @@ def add_field(self, model, field): if not self.skip_default(field) and self.effective_default(field) is not None: changes_sql, params = self._alter_column_default_sql(model, None, field, drop=True) sql = self.sql_alter_column % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(table), "changes": changes_sql, } self.execute(sql, params) @@ -934,8 +947,7 @@ def create_unique_name(*args, **kwargs): compiler = Query(model, alias_cols=False).get_compiler(connection=self.connection) columns = [field.column for field in fields] - table = model._meta.db_table - + table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) if name is None: name = IndexName(table, columns, '_uniq', create_unique_name) else: @@ -981,10 +993,10 @@ def _create_unique_sql(self, model, columns, def create_unique_name(*args, **kwargs): return self.quote_name(self._create_index_name(*args, **kwargs)) - - table = Table(model._meta.db_table, self.quote_name) + db_table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) + table = Table(db_table, self.quote_name) if name is None: - name = IndexName(model._meta.db_table, columns, '_uniq', create_unique_name) + name = IndexName(db_table, columns, '_uniq', create_unique_name) else: name = self.quote_name(name) columns = Columns(table, columns, self.quote_name) @@ -1022,16 +1034,58 @@ def _create_index_sql(self, model, fields, *, name=None, suffix='', using='', indexes, ...). """ if django_version >= (3, 2): - return super()._create_index_sql( - model, fields=fields, name=name, suffix=suffix, using=using, - db_tablespace=db_tablespace, col_suffixes=col_suffixes, sql=sql, - opclasses=opclasses, condition=condition, include=include, - expressions=expressions, + fields = fields or [] + expressions = expressions or [] + compiler = Query(model, alias_cols=False).get_compiler( + connection=self.connection, + ) + tablespace_sql = self._get_index_tablespace_sql( + model, fields, db_tablespace=db_tablespace + ) + columns = [field.column for field in fields] + sql_create_index = sql or self.sql_create_index + table_name = model._meta.db_table + table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) + def create_index_name(*args, **kwargs): + nonlocal name + if name is None: + name = self._create_index_name(*args, **kwargs) + return self.quote_name(name) + + return Statement( + sql_create_index, + table=Table(table, self.quote_name), + name=IndexName(table_name, columns, suffix, create_index_name), + using=using, + columns=( + self._index_columns(table, columns, col_suffixes, opclasses) + if columns + else Expressions(table, expressions, compiler, self.quote_value) + ), + extra=tablespace_sql, + condition=self._index_condition_sql(condition), + include=self._index_include_sql(model, include), ) - return super()._create_index_sql( - model, fields=fields, name=name, suffix=suffix, using=using, - db_tablespace=db_tablespace, col_suffixes=col_suffixes, sql=sql, - opclasses=opclasses, condition=condition, + table_name = model._meta.db_table + table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) + tablespace_sql = self._get_index_tablespace_sql(model, fields, db_tablespace=db_tablespace) + columns = [field.column for field in fields] + sql_create_index = sql or self.sql_create_index + + def create_index_name(*args, **kwargs): + nonlocal name + if name is None: + name = self._create_index_name(*args, **kwargs) + return self.quote_name(name) + + return Statement( + sql_create_index, + table=Table(table, self.quote_name), + name=IndexName(table_name, columns, suffix, create_index_name), + using=using, + columns=self._index_columns(table, columns, col_suffixes, opclasses), + extra=tablespace_sql, + condition=(' WHERE ' + condition) if condition else '', ) def create_model(self, model): @@ -1040,6 +1094,8 @@ def create_model(self, model): Will also create any accompanying indexes or unique constraints. """ # Create column SQL, add FK deferreds if needed + table_name = model._meta.db_table + db_table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) column_sqls = [] params = [] for field in model._meta.local_fields: @@ -1063,7 +1119,7 @@ def create_model(self, model): if db_params['check']: # SQL Server requires a name for the check constraint definition += self._sql_check_constraint % { - "name": self._create_index_name(model._meta.db_table, [field.column], suffix="_check"), + "name": self._create_index_name(table_name, [field.column], suffix="_check"), "check": db_params['check'] } # Autoincrement SQL (for backends with inline variant) @@ -1089,7 +1145,7 @@ def create_model(self, model): )) # Autoincrement SQL (for backends with post table definition variant) if field.get_internal_type() in ("AutoField", "BigAutoField", "SmallAutoField"): - autoinc_sql = self.connection.ops.autoinc_sql(model._meta.db_table, field.column) + autoinc_sql = self.connection.ops.autoinc_sql(db_table, field.column) if autoinc_sql: self.deferred_sql.extend(autoinc_sql) @@ -1107,7 +1163,7 @@ def create_model(self, model): constraints = [constraint.constraint_sql(model, self) for constraint in model._meta.constraints] # Make the table sql = self.sql_create_table % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(db_table), 'definition': ', '.join(constraint for constraint in (*column_sqls, *constraints) if constraint), } if model._meta.db_tablespace: @@ -1153,7 +1209,24 @@ def _delete_unique_sql( return self._delete_constraint_sql(sql, model, name) def delete_model(self, model): - super().delete_model(model) + """Delete a model from the database.""" + # Handle auto-created intermediary models + db_table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) + for field in model._meta.local_many_to_many: + if field.remote_field.through._meta.auto_created: + self.delete_model(field.remote_field.through) + + # Delete the table + self.execute( + self.sql_delete_table + % { + "table": self.quote_name(db_table), + } + ) + # Remove all deferred statements referencing the deleted table. + for sql in list(self.deferred_sql): + if isinstance(sql, Statement) and sql.references_table(db_table): + self.deferred_sql.remove(sql) def execute(self, sql, params=(), has_result=False): """ @@ -1210,7 +1283,38 @@ def quote_value(self, value): return "1" if value else "0" else: return str(value) - + def _create_fk_sql(self, model, field, suffix): + table_name = model._meta.db_table + db_table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) + to_db_table = get_table_name(self, field.target_field.model._meta.db_table, getattr(field.target_field.model._meta, "db_table_schema", False)) + table = Table(db_table, self.quote_name) + name = self._fk_constraint_name(model, field, suffix) + column = Columns(table_name, [field.column], self.quote_name) + to_table = Table(to_db_table, self.quote_name) + to_column = Columns(field.target_field.model._meta.db_table, [field.target_field.column], self.quote_name) + deferrable = self.connection.ops.deferrable_sql() + return Statement( + self.sql_create_fk, + table=table, + name=name, + column=column, + to_table=to_table, + to_column=to_column, + deferrable=deferrable, + ) + def _fk_constraint_name(self, model, field, suffix): + table_name = model._meta.db_table + to_db_table = get_table_name(self, field.target_field.model._meta.db_table, getattr(field.target_field.model._meta, "db_table_schema", False)) + def create_fk_name(*args, **kwargs): + return self.quote_name(self._create_index_name(*args, **kwargs)) + return ForeignKeyName( + table_name, + [field.column], + to_db_table, + [field.target_field.column], + suffix, + create_fk_name, + ) def remove_field(self, model, field): """ Removes a field from a model. Usually involves deleting a column, @@ -1225,6 +1329,7 @@ def remove_field(self, model, field): # Drop any FK constraints, SQL Server requires explicit deletion with self.connection.cursor() as cursor: constraints = self.connection.introspection.get_constraints(cursor, model._meta.db_table) + db_table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) for name, infodict in constraints.items(): if field.column in infodict['columns'] and infodict['foreign_key']: self.execute(self._delete_constraint_sql(self.sql_delete_fk, model, name)) @@ -1232,21 +1337,21 @@ def remove_field(self, model, field): for name, infodict in constraints.items(): if field.column in infodict['columns'] and infodict['index']: self.execute(self.sql_delete_index % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(db_table), "name": self.quote_name(name), }) # Drop primary key constraint, SQL Server requires explicit deletion for name, infodict in constraints.items(): if field.column in infodict['columns'] and infodict['primary_key']: self.execute(self.sql_delete_pk % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(db_table), "name": self.quote_name(name), }) # Drop check constraints, SQL Server requires explicit deletion for name, infodict in constraints.items(): if field.column in infodict['columns'] and infodict['check']: self.execute(self.sql_delete_check % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(db_table), "name": self.quote_name(name), }) # Drop unique constraints, SQL Server requires explicit deletion @@ -1254,12 +1359,12 @@ def remove_field(self, model, field): if (field.column in infodict['columns'] and infodict['unique'] and not infodict['primary_key'] and not infodict['index']): self.execute(self.sql_delete_unique % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(db_table), "name": self.quote_name(name), }) # Delete the column sql = self.sql_delete_column % { - "table": self.quote_name(model._meta.db_table), + "table": self.quote_name(db_table), "column": self.quote_name(field.column), } self.execute(sql) @@ -1268,7 +1373,7 @@ def remove_field(self, model, field): self.connection.close() # Remove all deferred statements referencing the deleted column. for sql in list(self.deferred_sql): - if isinstance(sql, Statement) and sql.references_column(model._meta.db_table, field.column): + if isinstance(sql, Statement) and sql.references_column(db_table, field.column): self.deferred_sql.remove(sql) def add_constraint(self, model, constraint): @@ -1291,3 +1396,11 @@ def _create_index_name(self, table_name, column_names, suffix=""): new_index_name = index_name.replace('[', '').replace(']', '').replace('.', '_') return new_index_name return index_name + + def _delete_constraint_sql(self, template, model, name): + table = get_table_name(self, model._meta.db_table, getattr(model._meta, "db_table_schema", False)) + return Statement( + template, + table=Table(table, self.quote_name), + name=self.quote_name(name), + ) diff --git a/testapp/migrations/0001_initial.py b/testapp/migrations/0001_initial.py index 6d898a4d..b68cfbda 100644 --- a/testapp/migrations/0001_initial.py +++ b/testapp/migrations/0001_initial.py @@ -1,10 +1,14 @@ # Generated by Django 2.2.8.dev20191112211527 on 2019-11-15 01:38 import uuid - from django.db import migrations, models import django +def forwards(apps, schema_editor): + if not schema_editor.connection.vendor == 'microsoft': + return + schema_editor.execute("CREATE SCHEMA test_schema;") + class Migration(migrations.Migration): @@ -12,7 +16,6 @@ class Migration(migrations.Migration): dependencies = [ ] - operations = [ migrations.CreateModel( name='Author', @@ -21,12 +24,17 @@ class Migration(migrations.Migration): ('name', models.CharField(max_length=100)), ], ), + migrations.RunPython(forwards), migrations.CreateModel( name='Editor', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=100)), ], + options={ + 'db_table': 'editor', + 'db_table_schema': 'test_schema' + }, ), migrations.CreateModel( name='Post', diff --git a/testapp/models.py b/testapp/models.py index f92d10f2..9fb9fe50 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the BSD license. +import django.db.models.options as options +options.DEFAULT_NAMES = options.DEFAULT_NAMES + ('db_table_schema',) import datetime import uuid @@ -19,11 +21,12 @@ class Meta: class Author(models.Model): name = models.CharField(max_length=100) - class Editor(BigAutoFieldMixin, models.Model): name = models.CharField(max_length=100) - - + class Meta: + db_table = 'editor' + db_table_schema = 'test_schema' + class Post(BigAutoFieldMixin, models.Model): title = models.CharField('title', max_length=255) author = models.ForeignKey(Author, models.CASCADE) @@ -70,7 +73,6 @@ class TestUniqueNullableModel(models.Model): # but for a unique index (not db_index) y_renamed = models.IntegerField(null=True, unique=True) - class TestNullableUniqueTogetherModel(models.Model): class Meta: unique_together = (('a', 'b', 'c'),) diff --git a/testapp/settings.py b/testapp/settings.py index ecdafbde..0e105a47 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -15,6 +15,7 @@ "PASSWORD": "MyPassword42", "HOST": "localhost", "PORT": "1433", + "SCHEMA" :"dbo", "OPTIONS": {"driver": "ODBC Driver 17 for SQL Server", }, }, 'other': { diff --git a/testapp/tests/test_queries.py b/testapp/tests/test_queries.py index 1f182d7d..4c77ed0d 100644 --- a/testapp/tests/test_queries.py +++ b/testapp/tests/test_queries.py @@ -17,7 +17,7 @@ def test_insert_into_table_with_trigger(self): ON [testapp_author] FOR INSERT AS - INSERT INTO [testapp_editor]([name]) VALUES ('Bar') + INSERT INTO [test_schema].[editor]([name]) VALUES ('Bar') """) try: From 0a895a886e8c39f9169f7c0c1a6d908b75961ab9 Mon Sep 17 00:00:00 2001 From: nasreddine27 Date: Tue, 20 Jun 2023 15:56:01 +0200 Subject: [PATCH 25/38] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index d4b30afa..60739f55 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,11 @@ in DATABASES control the behavior of the backend: - PASSWORD String. Database user password. + +- SCHEMA + String. Default schema to use. Not required. + - TOKEN String. Access token fetched as a user or service principal which From 05329e6b87411549c19d27089d2eca3ca3793e98 Mon Sep 17 00:00:00 2001 From: nasreddine27 Date: Mon, 24 Jul 2023 11:50:08 +0200 Subject: [PATCH 26/38] rename table fix --- mssql/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/schema.py b/mssql/schema.py index 55441f8d..c85e35fc 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -93,7 +93,7 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): DROP TABLE %(table)s """ sql_rename_column = "EXEC sp_rename '%(table)s.%(old_column)s', %(new_column)s, 'COLUMN'" - sql_rename_table = "EXEC sp_rename %(old_table)s, %(new_table)s" + sql_rename_table = "EXEC sp_rename '%(old_table)s', '%(new_table)s'" sql_create_unique_null = "CREATE UNIQUE INDEX %(name)s ON %(table)s(%(columns)s) " \ "WHERE %(columns)s IS NOT NULL" From 9c3be2b189e32c77696d64adf6729e94e5aadc6d Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Mon, 24 Jul 2023 20:57:13 -0700 Subject: [PATCH 27/38] add partial support for adding/altering comments on columns and tables --- mssql/schema.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 00ee5403..b171048d 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -93,6 +93,31 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_rename_table = "EXEC sp_rename %(old_table)s, %(new_table)s" sql_create_unique_null = "CREATE UNIQUE INDEX %(name)s ON %(table)s(%(columns)s) " \ "WHERE %(columns)s IS NOT NULL" + sql_alter_table_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.TABLES i + INNER JOIN sys.tables t ON t.name = i.TABLE_NAME + LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id + WHERE (ep.name = 'MS_Description' AND ep.minor_id = 0)) + EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, + @level0type = N'SCHEMA', @level0name = N'dbo', + @level1type = N'TABLE', @level1name = %(table)s + ELSE + EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, + @level0type = N'SCHEMA', @level0name = N'dbo', + @level1type = N'TABLE', @level1name = %(table)s;""" + + sql_alter_column_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.COLUMNS i + INNER JOIN sys.columns t ON t.name = i.COLUMN_NAME + LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id + WHERE (ep.name = N'MS_Description' AND ep.minor_id = column_id)) + EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, + @level0type = N'SCHEMA', @level0name = N'dbo', + @level1type = N'TABLE', @level1name = %(table)s, + @level2type = 'COLUMN', @level2name = %(column)s + ELSE + EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, + @level0type = N'SCHEMA', @level0name = N'dbo', + @level1type = N'TABLE', @level1name = %(table)s, + @level2type = 'COLUMN', @level2name = %(column)s; """ _deferred_unique_indexes = defaultdict(list) @@ -316,7 +341,11 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # Drop any FK constraints, we'll remake them later fks_dropped = set() - if old_field.remote_field and old_field.db_constraint: + if old_field.remote_field and old_field.db_constraint and self._field_should_be_altered( + old_field, + new_field, + ignore={"db_comment"}, + ): # Drop index, SQL Server requires explicit deletion if not hasattr(new_field, 'db_constraint') or not new_field.db_constraint: index_names = self._constraint_names(model, [old_field.column], index=True) @@ -447,7 +476,10 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, null_actions = [] post_actions = [] # Type change? - if old_type != new_type: + if old_type != new_type or ( + self.connection.features.supports_comments + and old_field.db_comment != new_field.db_comment + ): if django_version >= (4, 2): fragment, other_actions = self._alter_column_type_sql( model, old_field, new_field, new_type, old_collation=None, new_collation=None @@ -910,6 +942,18 @@ def add_field(self, model, field): "changes": changes_sql, } self.execute(sql, params) + # Add field comment, if required. + # if ( + # field.db_comment + # and self.connection.features.supports_comments + # and not self.connection.features.supports_comments_inline + # ): + # field_type = db_params["type"] + # self.execute( + # *self._alter_column_comment_sql( + # model, field, field_type, field.db_comment + # ) + # ) # Add an index, if required self.deferred_sql.extend(self._field_indexes_sql(model, field)) # Add any FK constraints later @@ -1117,6 +1161,23 @@ def create_model(self, model): # Prevent using [] as params, in the case a literal '%' is used in the definition self.execute(sql, params or None) + if self.connection.features.supports_comments: + # Add table comment. + if model._meta.db_table_comment: + self.alter_db_table_comment(model, None, model._meta.db_table_comment) + # Add column comments. + # if not self.connection.features.supports_comments_inline: + # for field in model._meta.local_fields: + # if field.db_comment: + # field_db_params = field.db_parameters( + # connection=self.connection + # ) + # field_type = field_db_params["type"] + # self.execute( + # *self._alter_column_comment_sql( + # model, field, field_type, field.db_comment + # ) + # ) # Add any field index and index_together's (deferred as SQLite3 _remake_table needs it) self.deferred_sql.extend(self._model_indexes_sql(model)) self.deferred_sql = list(set(self.deferred_sql)) @@ -1291,3 +1352,6 @@ def _create_index_name(self, table_name, column_names, suffix=""): new_index_name = index_name.replace('[', '').replace(']', '').replace('.', '_') return new_index_name return index_name + + def _alter_column_comment_sql(self, model, new_field, new_type, new_db_comment): + return "", [] \ No newline at end of file From be3616124d49e8f342c211caccc5ef4410414afc Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Thu, 27 Jul 2023 12:51:33 -0700 Subject: [PATCH 28/38] fix sql --- mssql/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index b171048d..6483bdc3 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -108,16 +108,16 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_alter_column_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.COLUMNS i INNER JOIN sys.columns t ON t.name = i.COLUMN_NAME LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id - WHERE (ep.name = N'MS_Description' AND ep.minor_id = column_id)) + WHERE (ep.name = N'MS_Description' AND ep.minor_id = 2)) EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = %(table)s, - @level2type = 'COLUMN', @level2name = %(column)s + @level2type = N'COLUMN', @level2name = %(column)s ELSE EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = %(table)s, - @level2type = 'COLUMN', @level2name = %(column)s; """ + @level2type = N'COLUMN', @level2name = %(column)s; """ _deferred_unique_indexes = defaultdict(list) From 8f9f00a09f13e8f8cda3c90a1cf23c426bd5f92e Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Thu, 27 Jul 2023 13:02:36 -0700 Subject: [PATCH 29/38] add django 4.2 condition and fix sql --- mssql/schema.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 6483bdc3..6899c2df 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -105,19 +105,10 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = %(table)s;""" - sql_alter_column_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.COLUMNS i - INNER JOIN sys.columns t ON t.name = i.COLUMN_NAME - LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id - WHERE (ep.name = N'MS_Description' AND ep.minor_id = 2)) - EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, + sql_alter_column_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = %(table)s, - @level2type = N'COLUMN', @level2name = %(column)s - ELSE - EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, - @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s, - @level2type = N'COLUMN', @level2name = %(column)s; """ + @level2type = N'COLUMN', @level2name = %(column)s""" _deferred_unique_indexes = defaultdict(list) @@ -341,11 +332,11 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # Drop any FK constraints, we'll remake them later fks_dropped = set() - if old_field.remote_field and old_field.db_constraint and self._field_should_be_altered( + if old_field.remote_field and old_field.db_constraint and (django_version >= (4, 2) and self._field_should_be_altered( old_field, new_field, ignore={"db_comment"}, - ): + )): # Drop index, SQL Server requires explicit deletion if not hasattr(new_field, 'db_constraint') or not new_field.db_constraint: index_names = self._constraint_names(model, [old_field.column], index=True) @@ -475,8 +466,8 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, actions = [] null_actions = [] post_actions = [] - # Type change? - if old_type != new_type or ( + # Type or comment change? + if old_type != new_type or (django_version >= (4, 2) and self.connection.features.supports_comments and old_field.db_comment != new_field.db_comment ): @@ -1161,7 +1152,7 @@ def create_model(self, model): # Prevent using [] as params, in the case a literal '%' is used in the definition self.execute(sql, params or None) - if self.connection.features.supports_comments: + if django_version >= (4, 2) and self.connection.features.supports_comments: # Add table comment. if model._meta.db_table_comment: self.alter_db_table_comment(model, None, model._meta.db_table_comment) From f9190bec47b8d3d2d8cb2d52a669722fdf706033 Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Thu, 27 Jul 2023 13:46:35 -0700 Subject: [PATCH 30/38] fix sql --- mssql/schema.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 6899c2df..7f0bf153 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -93,17 +93,9 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_rename_table = "EXEC sp_rename %(old_table)s, %(new_table)s" sql_create_unique_null = "CREATE UNIQUE INDEX %(name)s ON %(table)s(%(columns)s) " \ "WHERE %(columns)s IS NOT NULL" - sql_alter_table_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.TABLES i - INNER JOIN sys.tables t ON t.name = i.TABLE_NAME - LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id - WHERE (ep.name = 'MS_Description' AND ep.minor_id = 0)) - EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, + sql_alter_table_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s - ELSE - EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, - @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s;""" + @level1type = N'TABLE', @level1name = %(table)s""" sql_alter_column_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', From 25900238b2c5b856dec6973dbb9e00e40aee6c4d Mon Sep 17 00:00:00 2001 From: "Daniel Au (SIMBA TECHNOLOGIES INC)" Date: Thu, 7 Dec 2023 17:49:49 -0800 Subject: [PATCH 31/38] Implement db_comment --- mssql/introspection.py | 2 +- mssql/schema.py | 94 ++++++++++++++++++++++++++---------------- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index 16381fdf..86f217e5 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -83,7 +83,7 @@ def get_table_list(self, cursor): INNER JOIN sys.tables t ON t.name = i.TABLE_NAME LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id WHERE - ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id = 0) OR ep.value IS NULL) + ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id >= 0) OR ep.value IS NULL) AND i.TABLE_SCHEMA = %s""" % ( get_schema_name()) diff --git a/mssql/schema.py b/mssql/schema.py index 7f0bf153..35ea60e8 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -93,15 +93,40 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_rename_table = "EXEC sp_rename %(old_table)s, %(new_table)s" sql_create_unique_null = "CREATE UNIQUE INDEX %(name)s ON %(table)s(%(columns)s) " \ "WHERE %(columns)s IS NOT NULL" - sql_alter_table_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, - @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s""" - - sql_alter_column_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, - @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s, - @level2type = N'COLUMN', @level2name = %(column)s""" - + sql_alter_table_comment= """ + IF NOT EXISTS (SELECT NULL FROM sys.extended_properties ep + WHERE ep.major_id = OBJECT_ID('%(table)s') + AND ep.name = 'MS_Description' + AND ep.minor_id = 0) + EXECUTE sp_addextendedproperty + @name = 'MS_Description', @value = %(comment)s, + @level0type = 'SCHEMA', @level0name = 'dbo', + @level1type = 'TABLE', @level1name = %(table)s + ELSE + EXECUTE sp_updateextendedproperty + @name = 'MS_Description', @value = %(comment)s, + @level0type = 'SCHEMA', @level0name = 'dbo', + @level1type = 'TABLE', @level1name = %(table)s + """ + sql_alter_column_comment= """ + IF NOT EXISTS (SELECT NULL FROM sys.extended_properties ep + WHERE ep.major_id = OBJECT_ID('%(table)s') + AND ep.name = 'MS_Description' + AND ep.minor_id = (SELECT column_id FROM sys.columns + WHERE name = '%(column)s' + AND object_id = OBJECT_ID('%(table)s'))) + EXECUTE sp_addextendedproperty + @name = 'MS_Description', @value = %(comment)s, + @level0type = 'SCHEMA', @level0name = 'dbo', + @level1type = 'TABLE', @level1name = %(table)s, + @level2type = 'COLUMN', @level2name = %(column)s + ELSE + EXECUTE sp_updateextendedproperty + @name = 'MS_Description', @value = %(comment)s, + @level0type = 'SCHEMA', @level0name = 'dbo', + @level1type = 'TABLE', @level1name = %(table)s, + @level2type = 'COLUMN', @level2name = %(column)s + """ _deferred_unique_indexes = defaultdict(list) def _alter_column_default_sql(self, model, old_field, new_field, drop=False): @@ -172,6 +197,8 @@ def _alter_column_null_sql(self, model, old_field, new_field): if django_version >= (4, 2): def _alter_column_type_sql(self, model, old_field, new_field, new_type, old_collation, new_collation): new_type = self._set_field_new_type_null_status(old_field, new_type) + # Check if existing + # Drop exisiting return super()._alter_column_type_sql(model, old_field, new_field, new_type, old_collation, new_collation) else: def _alter_column_type_sql(self, model, old_field, new_field, new_type): @@ -926,17 +953,17 @@ def add_field(self, model, field): } self.execute(sql, params) # Add field comment, if required. - # if ( - # field.db_comment - # and self.connection.features.supports_comments - # and not self.connection.features.supports_comments_inline - # ): - # field_type = db_params["type"] - # self.execute( - # *self._alter_column_comment_sql( - # model, field, field_type, field.db_comment - # ) - # ) + if ( + field.db_comment + and self.connection.features.supports_comments + and not self.connection.features.supports_comments_inline + ): + field_type = db_params["type"] + self.execute( + *self._alter_column_comment_sql( + model, field, field_type, field.db_comment + ) + ) # Add an index, if required self.deferred_sql.extend(self._field_indexes_sql(model, field)) # Add any FK constraints later @@ -1149,18 +1176,18 @@ def create_model(self, model): if model._meta.db_table_comment: self.alter_db_table_comment(model, None, model._meta.db_table_comment) # Add column comments. - # if not self.connection.features.supports_comments_inline: - # for field in model._meta.local_fields: - # if field.db_comment: - # field_db_params = field.db_parameters( - # connection=self.connection - # ) - # field_type = field_db_params["type"] - # self.execute( - # *self._alter_column_comment_sql( - # model, field, field_type, field.db_comment - # ) - # ) + if not self.connection.features.supports_comments_inline: + for field in model._meta.local_fields: + if field.db_comment: + field_db_params = field.db_parameters( + connection=self.connection + ) + field_type = field_db_params["type"] + self.execute( + *self._alter_column_comment_sql( + model, field, field_type, field.db_comment + ) + ) # Add any field index and index_together's (deferred as SQLite3 _remake_table needs it) self.deferred_sql.extend(self._model_indexes_sql(model)) self.deferred_sql = list(set(self.deferred_sql)) @@ -1335,6 +1362,3 @@ def _create_index_name(self, table_name, column_names, suffix=""): new_index_name = index_name.replace('[', '').replace(']', '').replace('.', '_') return new_index_name return index_name - - def _alter_column_comment_sql(self, model, new_field, new_type, new_db_comment): - return "", [] \ No newline at end of file From 965573a80695a12da35417fec5535080aa8eb928 Mon Sep 17 00:00:00 2001 From: "Daniel Au (SIMBA TECHNOLOGIES INC)" Date: Fri, 8 Dec 2023 10:52:14 -0800 Subject: [PATCH 32/38] Update get_table_list and add version condition to db_comment --- mssql/introspection.py | 8 +++----- mssql/schema.py | 23 +++++++++++------------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index 713dd07e..efba52ad 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -82,12 +82,10 @@ def get_table_list(self, cursor): TABLE_TYPE, CAST(ep.value AS VARCHAR) AS COMMENT FROM INFORMATION_SCHEMA.TABLES i - INNER JOIN sys.tables t ON t.name = i.TABLE_NAME + LEFT JOIN sys.tables t ON t.name = i.TABLE_NAME LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id - WHERE - ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id >= 0) OR ep.value IS NULL) - AND - i.TABLE_SCHEMA = %s""" % ( + AND ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id = 0) OR ep.value IS NULL) + AND i.TABLE_SCHEMA = %s""" % ( get_schema_name()) cursor.execute(sql) types = {'BASE TABLE': 't', 'VIEW': 'v'} diff --git a/mssql/schema.py b/mssql/schema.py index 34d347a7..1768b4f9 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -197,8 +197,6 @@ def _alter_column_null_sql(self, model, old_field, new_field): if django_version >= (4, 2): def _alter_column_type_sql(self, model, old_field, new_field, new_type, old_collation, new_collation): new_type = self._set_field_new_type_null_status(old_field, new_type) - # Check if existing - # Drop exisiting return super()._alter_column_type_sql(model, old_field, new_field, new_type, old_collation, new_collation) else: def _alter_column_type_sql(self, model, old_field, new_field, new_type): @@ -965,17 +963,18 @@ def add_field(self, model, field): } self.execute(sql, params) # Add field comment, if required. - if ( - field.db_comment - and self.connection.features.supports_comments - and not self.connection.features.supports_comments_inline - ): - field_type = db_params["type"] - self.execute( - *self._alter_column_comment_sql( - model, field, field_type, field.db_comment + if django_version >= (4, 2): + if ( + field.db_comment + and self.connection.features.supports_comments + and not self.connection.features.supports_comments_inline + ): + field_type = db_params["type"] + self.execute( + *self._alter_column_comment_sql( + model, field, field_type, field.db_comment + ) ) - ) # Add an index, if required self.deferred_sql.extend(self._field_indexes_sql(model, field)) # Add any FK constraints later From 37d48d0bee0c8005b904f0e5e3347bec5e83cafb Mon Sep 17 00:00:00 2001 From: "Daniel Au (SIMBA TECHNOLOGIES INC)" Date: Mon, 11 Dec 2023 10:47:32 -0800 Subject: [PATCH 33/38] Fix drop fk condition --- mssql/schema.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 1768b4f9..42b64ec5 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -349,11 +349,19 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # Drop any FK constraints, we'll remake them later fks_dropped = set() - if old_field.remote_field and old_field.db_constraint and (django_version >= (4, 2) and self._field_should_be_altered( - old_field, - new_field, - ignore={"db_comment"}, - )): + if ( + old_field.remote_field + and old_field.db_constraint + and (django_version < (4,2) + or + (django_version >= (4, 2) + and self._field_should_be_altered( + old_field, + new_field, + ignore={"db_comment"}) + ) + ) + ): # Drop index, SQL Server requires explicit deletion if not hasattr(new_field, 'db_constraint') or not new_field.db_constraint: index_names = self._constraint_names(model, [old_field.column], index=True) From bb2cb087e2b506d6a5671deff820fc55f2fc8609 Mon Sep 17 00:00:00 2001 From: "Daniel Au (SIMBA TECHNOLOGIES INC)" Date: Mon, 11 Dec 2023 14:42:24 -0800 Subject: [PATCH 34/38] Fix alter comment behavior --- mssql/schema.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mssql/schema.py b/mssql/schema.py index 42b64ec5..771b93b6 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -171,7 +171,18 @@ def _alter_column_default_sql(self, model, old_field, new_field, drop=False): }, params, ) - + + def _alter_column_comment_sql(self, model, new_field, new_type, new_db_comment): + return ( + self.sql_alter_column_comment + % { + "table": self.quote_name(model._meta.db_table), + "column": new_field.column, + "comment": self._comment_sql(new_db_comment), + }, + [], + ) + def _alter_column_null_sql(self, model, old_field, new_field): """ Hook to specialize column null alteration. From e11aaa7a65bacefbb6a046d2a85eba4c1270b669 Mon Sep 17 00:00:00 2001 From: nasreddine27 Date: Fri, 7 Jun 2024 10:59:31 +0200 Subject: [PATCH 35/38] Update settings.py to Allow db_table_schema to be added on migration files --- testapp/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testapp/settings.py b/testapp/settings.py index 403975d6..9a67cadd 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -5,6 +5,8 @@ from django import VERSION +import django.db.models.options as options + BASE_DIR = Path(__file__).resolve().parent.parent DATABASES = { @@ -301,3 +303,6 @@ 'model_fields.test_jsonfield.TestQuerying.test_key_iregex', 'model_fields.test_jsonfield.TestQuerying.test_key_regex', ] + +if not 'db_table_schema' in options.DEFAULT_NAMES : + options.DEFAULT_NAMES = options.DEFAULT_NAMES + ('db_table_schema',) From 8dfffab607f4dce102163d8a8f0c8220872227af Mon Sep 17 00:00:00 2001 From: nasreddine27 Date: Fri, 7 Jun 2024 10:59:55 +0200 Subject: [PATCH 36/38] Update models.py --- testapp/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/testapp/models.py b/testapp/models.py index eda2a8d6..0d0a0fa4 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -1,8 +1,6 @@ # Copyright (c) Microsoft Corporation. # Licensed under the BSD license. -import django.db.models.options as options -options.DEFAULT_NAMES = options.DEFAULT_NAMES + ('db_table_schema',) import datetime import uuid From 0c2d035301cbd6e13ab2f4825704f7424159f9b3 Mon Sep 17 00:00:00 2001 From: nasreddine27 Date: Fri, 7 Jun 2024 11:03:33 +0200 Subject: [PATCH 37/38] Update compiler.py : treat each model separately when a a JOIN clause is used The problem detected when there is a JOIN clause and the schema of the joined table is different from the parent one. the solution is to treat each model separately. --- mssql/compiler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mssql/compiler.py b/mssql/compiler.py index 3c071ccf..7788a3ab 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -19,6 +19,7 @@ from django.core.exceptions import EmptyResultSet, FullResultSet from .introspection import get_table_name, get_schema_name +from django.apps import apps def _as_sql_agv(self, compiler, connection): return self.as_sql(compiler, connection, template='%(function)s(CONVERT(float, %(field)s))') @@ -460,10 +461,10 @@ def get_from_clause(self): # Extra tables can end up in self.tables, but not in the # alias_map if they aren't in a join. That's OK. We skip them. continue - opts = self.query.get_meta() settings_dict = self.connection.settings_dict - schema = getattr(opts, "db_table_schema", settings_dict.get('SCHEMA', False)) clause_sql, clause_params = self.compile(from_clause) + model = next((m for m in apps.get_models() if m._meta.db_table == from_clause.table_name), None) + schema = getattr(getattr(model,"_meta", None), "db_table_schema", settings_dict.get('SCHEMA', False)) if schema: if 'JOIN' in clause_sql: table_clause_sql = clause_sql.split('JOIN ')[1].split(' ON')[0] From e2f086b7fd4efc22430fe25bba5f31e18f0467fa Mon Sep 17 00:00:00 2001 From: nasreddine27 Date: Thu, 20 Feb 2025 10:56:37 +0100 Subject: [PATCH 38/38] Update 0001_initial.py add comments --- testapp/migrations/0001_initial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testapp/migrations/0001_initial.py b/testapp/migrations/0001_initial.py index b68cfbda..4feac24e 100644 --- a/testapp/migrations/0001_initial.py +++ b/testapp/migrations/0001_initial.py @@ -5,6 +5,7 @@ import django def forwards(apps, schema_editor): + #create schema for testing purpose if not schema_editor.connection.vendor == 'microsoft': return schema_editor.execute("CREATE SCHEMA test_schema;")