diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/15_plus/alter_index_no_depends.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/15_plus/alter_index_no_depends.sql index eb0be343a58..f05567f540d 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/15_plus/alter_index_no_depends.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/15_plus/alter_index_no_depends.sql @@ -7,7 +7,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS "Idx_$%{}[]()&*^!@""'`\/#" (id ASC NULLS LAST, name COLLATE pg_catalog."POSIX" text_pattern_ops ASC NULLS LAST) INCLUDE(name, id) NULLS NOT DISTINCT - WITH (fillfactor=10, deduplicate_items=False) + WITH (fillfactor=10, deduplicate_items=false) WHERE id < 100; diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/15_plus/create_index_with_ext.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/15_plus/create_index_with_ext.sql index d46299198e6..a4cbb386da0 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/15_plus/create_index_with_ext.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/15_plus/create_index_with_ext.sql @@ -7,7 +7,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS "Idx_$%{}[]()&*^!@""'`\/#" (id ASC NULLS LAST, name COLLATE pg_catalog."POSIX" text_pattern_ops ASC NULLS LAST) INCLUDE(name, id) NULLS NOT DISTINCT - WITH (fillfactor=10, deduplicate_items=False) + WITH (fillfactor=10, deduplicate_items=false) WHERE id < 100; diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/alter_index_no_depends.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/alter_index_no_depends.sql index 8284142636a..11d57949e01 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/alter_index_no_depends.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/alter_index_no_depends.sql @@ -6,7 +6,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS "Idx_$%{}[]()&*^!@""'`\/#" ON public.test_table_for_indexes USING btree (id ASC NULLS LAST, name COLLATE pg_catalog."POSIX" text_pattern_ops ASC NULLS LAST) INCLUDE(name, id) - WITH (fillfactor=10, deduplicate_items=False) + WITH (fillfactor=10, deduplicate_items=false) WHERE id < 100; diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_asc_null_first.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_asc_null_first.sql index 706e32ff508..cb153b22160 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_asc_null_first.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_asc_null_first.sql @@ -6,7 +6,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS "Idx_$%{}[]()&*^!@""'`\/#" ON public.test_table_for_indexes USING btree (id ASC NULLS FIRST, name COLLATE pg_catalog."POSIX" text_pattern_ops ASC NULLS FIRST) INCLUDE(name, id) - WITH (fillfactor=10, deduplicate_items=False) + WITH (fillfactor=10, deduplicate_items=false) WHERE id < 100; diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_asc_null_last.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_asc_null_last.sql index c69bf1ae964..5e5fe39c288 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_asc_null_last.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_asc_null_last.sql @@ -6,7 +6,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS "Idx_$%{}[]()&*^!@""'`\/#" ON public.test_table_for_indexes USING btree (id ASC NULLS LAST, name COLLATE pg_catalog."POSIX" text_pattern_ops ASC NULLS LAST) INCLUDE(name, id) - WITH (fillfactor=10, deduplicate_items=False) + WITH (fillfactor=10, deduplicate_items=false) WHERE id < 100; diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_desc_null_last.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_desc_null_last.sql index c31cc5d03d4..2a5aa9180ff 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_desc_null_last.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_btree_desc_null_last.sql @@ -6,7 +6,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS "Idx_$%{}[]()&*^!@""'`\/#" ON public.test_table_for_indexes USING btree (id DESC NULLS LAST, name COLLATE pg_catalog."POSIX" text_pattern_ops DESC NULLS LAST) INCLUDE(name, id) - WITH (fillfactor=10, deduplicate_items=False) + WITH (fillfactor=10, deduplicate_items=false) WHERE id < 100; diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_index_with_ext.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_index_with_ext.sql index 27bddad38df..18cc4ec48ce 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_index_with_ext.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/default/create_index_with_ext.sql @@ -6,7 +6,7 @@ CREATE UNIQUE INDEX IF NOT EXISTS "Idx_$%{}[]()&*^!@""'`\/#" ON public.test_table_for_indexes USING btree (id ASC NULLS LAST, name COLLATE pg_catalog."POSIX" text_pattern_ops ASC NULLS LAST) INCLUDE(name, id) - WITH (fillfactor=10, deduplicate_items=False) + WITH (fillfactor=10, deduplicate_items=false) WHERE id < 100; diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/test_indexes_sql_options.py b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/test_indexes_sql_options.py new file mode 100644 index 00000000000..d42e14bdf10 --- /dev/null +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/tests/test_indexes_sql_options.py @@ -0,0 +1,175 @@ +########################################################################## +# +# pgAdmin 4 - PostgreSQL Tools +# +# Copyright (C) 2013 - 2026, The pgAdmin Development Team +# This software is released under the PostgreSQL Licence +# +########################################################################## + +"""Reverse engineered index SQL must preserve opclass options (e.g. +gist_trgm_ops(siglen=32)) and every storage parameter found in the +catalog, including ones pgAdmin does not know about. + +https://github.com/pgadmin-org/pgadmin4/issues/9625 +""" + +import re +import uuid + +from pgadmin.browser.server_groups.servers.databases.schemas.tables.columns. \ + tests import utils as columns_utils +from pgadmin.browser.server_groups.servers.databases.schemas.tables.tests \ + import utils as tables_utils +from pgadmin.browser.server_groups.servers.databases.schemas.tests import \ + utils as schema_utils +from pgadmin.browser.server_groups.servers.databases.tests import utils as \ + database_utils +from pgadmin.utils.route import BaseTestGenerator +from regression import parent_node_dict +from regression.python_test_utils import test_utils as utils + + +class IndexSqlOptionsTestCase(BaseTestGenerator): + """This class tests sql generated for indexes having opclass options + or storage parameters unknown to pgAdmin.""" + + url = "/browser/index/sql/" + scenarios = utils.generate_scenarios("index_sql_options", { + "index_sql_options": [{ + "name": "Index SQL keeps opclass options and all reloptions", + "is_positive_test": True, + "inventory_data": {}, + "test_data": {}, + "mocking_required": False, + "mock_data": {}, + "expected_data": { + "status_code": 200, + "error_msg": None, + "test_result_data": {} + }, + "is_list": False + }] + }) + + def setUp(self): + super().setUp() + self.db_name = parent_node_dict["database"][-1]["db_name"] + schema_info = parent_node_dict["schema"][-1] + self.server_id = schema_info["server_id"] + self.db_id = schema_info["db_id"] + db_con = database_utils.connect_database(self, utils.SERVER_GROUP, + self.server_id, self.db_id) + if not db_con['data']["connected"]: + raise Exception("Could not connect to database to add a table.") + self.schema_id = schema_info["schema_id"] + self.schema_name = schema_info["schema_name"] + schema_response = schema_utils.verify_schemas(self.server, + self.db_name, + self.schema_name) + if not schema_response: + raise Exception("Could not find the schema to add a table.") + self.table_name = "table_index_options_%s" % (str(uuid.uuid4())[1:8]) + self.table_id = tables_utils.create_table(self.server, self.db_name, + self.schema_name, + self.table_name) + self.column_name = "content" + columns_utils.create_column(self.server, self.db_name, + self.schema_name, self.table_name, + self.column_name, 'text') + + # Create the indexes with plain SQL: the index dialog supports + # neither opclass options nor arbitrary storage parameters, which + # is exactly what this test is about. + self.opclass_index_name = "idx_opclass_options_%s" % ( + str(uuid.uuid4())[1:8]) + self.relopts_index_name = "idx_all_reloptions_%s" % ( + str(uuid.uuid4())[1:8]) + connection = utils.get_db_connection(self.db_name, + self.server['username'], + self.server['db_password'], + self.server['host'], + self.server['port']) + old_isolation_level = connection.isolation_level + utils.set_isolation_level(connection, 0) + pg_cursor = connection.cursor() + pg_cursor.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") + pg_cursor.execute( + 'ALTER TABLE {0}.{1} ADD COLUMN search_vector tsvector'.format( + self.schema_name, self.table_name)) + pg_cursor.execute( + 'CREATE INDEX {0} ON {1}.{2} USING GIST ' + '({3} gist_trgm_ops(siglen=32))'.format( + self.opclass_index_name, self.schema_name, self.table_name, + self.column_name)) + pg_cursor.execute( + 'CREATE INDEX {0} ON {1}.{2} USING btree ({3}) ' + 'WITH (fillfactor=70, ' + 'vacuum_cleanup_index_scale_factor=0.5)'.format( + self.relopts_index_name, self.schema_name, self.table_name, + self.column_name)) + self.default_opclass_index_name = "idx_def_opclass_options_%s" % ( + str(uuid.uuid4())[1:8]) + pg_cursor.execute( + 'CREATE INDEX {0} ON {1}.{2} USING GIST ' + '(search_vector tsvector_ops(siglen=64))'.format( + self.default_opclass_index_name, self.schema_name, + self.table_name)) + pg_cursor.execute( + "SELECT cls.oid FROM pg_catalog.pg_class cls " + "JOIN pg_catalog.pg_namespace nsp ON nsp.oid = cls.relnamespace " + "WHERE cls.relkind = 'i' AND nsp.nspname = %s " + "AND cls.relname = ANY(%s)", + (self.schema_name, + [self.opclass_index_name, self.relopts_index_name, + self.default_opclass_index_name])) + self.index_ids = [row[0] for row in pg_cursor.fetchall()] + utils.set_isolation_level(connection, old_isolation_level) + connection.close() + if len(self.index_ids) != 3: + raise Exception("Could not create the test indexes.") + + def get_index_sql(self, index_id): + response = self.tester.get( + "{0}{1}/{2}/{3}/{4}/{5}/{6}".format(self.url, utils.SERVER_GROUP, + self.server_id, self.db_id, + self.schema_id, self.table_id, + index_id), + follow_redirects=True) + self.assertEqual(response.status_code, 200) + resp_sql = response.data.decode('unicode_escape') + if resp_sql.startswith('"') and resp_sql.endswith('"'): + resp_sql = resp_sql[1:-1] + # Collapse whitespace so assertions do not depend on template + # line breaks. + return re.sub(r'\s+', ' ', resp_sql) + + def runTest(self): + if self.is_positive_test: + sql_by_index = dict( + (idx, self.get_index_sql(idx)) for idx in self.index_ids) + opclass_sql = None + relopts_sql = None + for idx, sql in sql_by_index.items(): + if self.opclass_index_name in sql: + opclass_sql = sql + if self.relopts_index_name in sql: + relopts_sql = sql + self.assertIsNotNone(opclass_sql) + self.assertIsNotNone(relopts_sql) + self.assertIn('gist_trgm_ops(siglen=32)', opclass_sql) + # Options on the default opclass still require the opclass + # name to be rendered, otherwise the options are lost. + default_sql = None + for idx, sql in sql_by_index.items(): + if self.default_opclass_index_name in sql: + default_sql = sql + self.assertIsNotNone(default_sql) + self.assertIn('tsvector_ops(siglen=64)', default_sql) + self.assertIn('fillfactor=70', relopts_sql) + self.assertIn('vacuum_cleanup_index_scale_factor=0.5', + relopts_sql) + + def tearDown(self): + # Disconnect the database + database_utils.disconnect_database(self, self.server_id, self.db_id) diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/utils.py b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/utils.py index db6444f4c04..b9c5db2c27a 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/utils.py +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/indexes/utils.py @@ -129,6 +129,17 @@ def get_column_details(conn, idx, data, mode='properties', template_path=None): 'statistics': row['statistics'] } + # Preserve any opclass options (e.g. gist_trgm_ops(siglen=32)) in + # the generated SQL. Only when creating SQL: the properties + # dialog expects the plain opclass name. #9625 + if mode == 'create' and row['attoptions']: + # A default opclass has opcname=None, but the name must still + # be rendered for the options to survive. + opclass = row['opcname'] or row['opclassname'] + if opclass: + cols_data['op_class'] = "{0}({1})".format( + opclass, ', '.join(row['attoptions'])) + # ASC/DESC and NULLS works only with btree indexes if 'amname' in data and data['amname'] == 'btree': cols_data['sort_order'] = False @@ -334,11 +345,12 @@ def get_reverse_engineered_sql(conn, **kwargs): data['table'] = table data["storage_parameters"] = {} - storage_params = get_storage_params(data['amname']) - - for param in storage_params: - if (param in data) and (data[param] is not None): - data["storage_parameters"].update({param: data[param]}) + # Use the raw reloptions from the catalog so that every storage + # parameter is preserved, including ones pgAdmin doesn't know about + # (e.g. options of extension index access methods). #9625 + for option in data.get('reloptions') or []: + key, _, value = option.partition('=') + data["storage_parameters"][key] = value # Add column details for current index data = get_column_details(conn, idx, data, 'create') diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/15_plus/properties.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/15_plus/properties.sql index 2fce4aa3612..b735040a757 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/15_plus/properties.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/15_plus/properties.sql @@ -53,7 +53,8 @@ SELECT DISTINCT ON (cls.relname) substring(array_to_string(cls.reloptions, ',') FROM 'buffering=([a-z]*)') AS buffering, substring(array_to_string(cls.reloptions, ',') FROM 'fastupdate=([a-z]*)')::boolean AS fastupdate, substring(array_to_string(cls.reloptions, ',') FROM 'autosummarize=([a-z]*)')::boolean AS autosummarize, - substring(array_to_string(cls.reloptions, ',') FROM 'lists=([0-9]*)')::int AS lists + substring(array_to_string(cls.reloptions, ',') FROM 'lists=([0-9]*)')::int AS lists, + cls.reloptions {% if datlastsysoid %} , (cls.oid <= {{ datlastsysoid }}::oid) AS is_sys_idx {% endif %} diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/default/column_details.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/default/column_details.sql index cc53e6877c3..3a4529d1de9 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/default/column_details.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/default/column_details.sql @@ -11,6 +11,8 @@ SELECT pg_catalog.pg_get_indexdef(i.indexrelid, i.attnum, true) as attdef, CASE WHEN pg_catalog.pg_get_indexdef(i.indexrelid, i.attnum, true) = pg_catalog.quote_ident(a.attname) THEN FALSE ELSE TRUE END AS is_exp, a.attstattarget as statistics, + a.attoptions AS attoptions, + o.opcname AS opclassname, CASE WHEN (o.opcdefault = FALSE) THEN o.opcname ELSE null END AS opcname, op.oprname AS oprname, CASE WHEN length(nspc.nspname::text) > 0 AND length(coll.collname::text) > 0 THEN diff --git a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/default/properties.sql b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/default/properties.sql index 00b6c69558c..a26afe2696c 100644 --- a/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/default/properties.sql +++ b/web/pgadmin/browser/server_groups/servers/databases/schemas/tables/templates/indexes/sql/default/properties.sql @@ -52,7 +52,8 @@ SELECT DISTINCT ON (cls.relname) substring(array_to_string(cls.reloptions, ',') FROM 'buffering=([a-z]*)') AS buffering, substring(array_to_string(cls.reloptions, ',') FROM 'fastupdate=([a-z]*)') AS fastupdate, substring(array_to_string(cls.reloptions, ',') FROM 'autosummarize=([a-z]*)')::boolean AS autosummarize, - substring(array_to_string(cls.reloptions, ',') FROM 'lists=([0-9]*)')::int AS lists + substring(array_to_string(cls.reloptions, ',') FROM 'lists=([0-9]*)')::int AS lists, + cls.reloptions {% if datlastsysoid %} , (cls.oid <= {{ datlastsysoid }}::oid) AS is_sys_idx {% endif %}