Make SQL table creation scripts idempotent to support redeployment

DeploymentComponents/azure_sql_artifacts/README.md

@@ -1,4 +1,17 @@
-# Example Control Table Records
+# Azure SQL Artifacts
+
+## Deployment
+
+All SQL table creation scripts in the `tables/` directory are idempotent and can be safely run multiple times. This allows for:
+- Re-running deployments without errors
+- Updating existing databases with new schema changes
+- Supporting continuous deployment pipelines
+
+The scripts use `IF NOT EXISTS` checks before creating tables and adding constraints to ensure they don't fail when objects already exist in the database.
+
+**Note:** Stored procedures use `CREATE OR ALTER` syntax which is inherently idempotent.
+
+## Example Control Table Records
 
 ## Important Notes
 

DeploymentComponents/azure_sql_artifacts/tables/ControlTable.sql

@@ -7,6 +7,8 @@ GO
 SET QUOTED_IDENTIFIER ON
 GO
 
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[ControlTable]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[ControlTable](
 	[Id] [int] IDENTITY(1,1) NOT NULL,
 	[SourceObjectSettings] [nvarchar](max) NULL,
@@ -35,15 +37,22 @@ WITH
 (
 SYSTEM_VERSIONING = ON (HISTORY_TABLE = [dbo].[ControlTableHistory])
 )
-GO
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF_ControlTable_PurviewScanEnabled' AND parent_object_id = OBJECT_ID('[dbo].[ControlTable]'))
+BEGIN
 ALTER TABLE [dbo].[ControlTable] ADD  CONSTRAINT [DF_ControlTable_PurviewScanEnabled]  DEFAULT ((1)) FOR [PurviewScanEnabled]
-GO
+END
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF_ControlTable_ValidFrom' AND parent_object_id = OBJECT_ID('[dbo].[ControlTable]'))
+BEGIN
 ALTER TABLE [dbo].[ControlTable] ADD  CONSTRAINT [DF_ControlTable_ValidFrom]  DEFAULT (sysutcdatetime()) FOR [ValidFrom]
-GO
+END
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF_ControlTable_ValidTo' AND parent_object_id = OBJECT_ID('[dbo].[ControlTable]'))
+BEGIN
 ALTER TABLE [dbo].[ControlTable] ADD  CONSTRAINT [DF_ControlTable_ValidTo]  DEFAULT (CONVERT([datetime2],'9999-12-31 23:59:59.9999999')) FOR [ValidTo]
+END
+END
 GO
 
 DECLARE @controlTableRecords INTEGER 

DeploymentComponents/azure_sql_artifacts/tables/DataContract.sql

@@ -6,6 +6,9 @@ SET ANSI_NULLS ON
 GO
 SET QUOTED_IDENTIFIER ON
 GO
+
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[DataContract]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[DataContract](
 	[ID] [int] IDENTITY(1,1) NOT NULL,
 	[ContractID] [nvarchar](255) NOT NULL,
@@ -49,9 +52,13 @@ CREATE TABLE [dbo].[DataContract](
 	[DataAssetTechnicalInformation] [nvarchar](max) NULL,
 	[hsActive] [bit] NULL
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
-GO
+
+IF NOT EXISTS (SELECT * FROM sys.key_constraints WHERE name = 'PK__DataCont__3214EC27' AND parent_object_id = OBJECT_ID('[dbo].[DataContract]'))
+BEGIN
 ALTER TABLE [dbo].[DataContract] ADD PRIMARY KEY CLUSTERED 
 (
 	[ID] ASC
 )WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ONLINE = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
+END
+END
 GO

DeploymentComponents/azure_sql_artifacts/tables/DataMapping.sql

@@ -9,6 +9,8 @@ GO
 SET QUOTED_IDENTIFIER ON
 GO
 
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[DataMapping]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[DataMapping](
 	[id] [int] IDENTITY(1,1) NOT NULL,
 	[TimeStamp] [datetime2](7) NOT NULL,
@@ -38,14 +40,21 @@ CREATE TABLE [dbo].[DataMapping](
 	[id] ASC
 )WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
-GO
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF__DataMappi__activ__default' AND parent_object_id = OBJECT_ID('[dbo].[DataMapping]'))
+BEGIN
 ALTER TABLE [dbo].[DataMapping] ADD  DEFAULT (NULL) FOR [active]
-GO
+END
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF__DataMappi__activ__date1' AND parent_object_id = OBJECT_ID('[dbo].[DataMapping]'))
+BEGIN
 ALTER TABLE [dbo].[DataMapping] ADD  DEFAULT (getdate()) FOR [activeDate]
-GO
+END
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF__DataMappi__inact__date' AND parent_object_id = OBJECT_ID('[dbo].[DataMapping]'))
+BEGIN
 ALTER TABLE [dbo].[DataMapping] ADD  DEFAULT (getdate()) FOR [inactiveDate]
+END
+END
 GO
 

DeploymentComponents/azure_sql_artifacts/tables/Handshake.sql

@@ -9,6 +9,8 @@ GO
 SET QUOTED_IDENTIFIER ON
 GO
 
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[Handshake]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[Handshake](
 	[Id] [int] IDENTITY(1,1) NOT NULL,
 	[DataContractID] [nvarchar](200) NOT NULL,
@@ -33,4 +35,5 @@ PRIMARY KEY CLUSTERED
 	[Id] ASC
 )WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+END
 GO

DeploymentComponents/azure_sql_artifacts/tables/IgnoredLandedFiles.sql

@@ -6,10 +6,14 @@ SET ANSI_NULLS ON
 GO
 SET QUOTED_IDENTIFIER ON
 GO
+
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[IgnoredLandedFiles]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[IgnoredLandedFiles](
 	[filename] [nvarchar](255) NULL,
 	[folderpath] [nvarchar](255) NULL,
 	[condition] [nvarchar](255) NULL,
 	[reason] [nvarchar](1000) NULL
 ) ON [PRIMARY]
+END
 GO

DeploymentComponents/azure_sql_artifacts/tables/IngestedLandingDataAudit.sql

@@ -6,6 +6,9 @@ SET ANSI_NULLS ON
 GO
 SET QUOTED_IDENTIFIER ON
 GO
+
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[IngestedLandingDataAudit]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[id] [int] IDENTITY(1,1) NOT NULL,
 	[source_filename] [nvarchar](255) NULL,
@@ -33,10 +36,15 @@ WITH
 (
 SYSTEM_VERSIONING = ON (HISTORY_TABLE = [dbo].[IngestedLandingDataAuditHistory])
 )
-GO
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF_IngestedLandingDataAudit_ValidFrom' AND parent_object_id = OBJECT_ID('[dbo].[IngestedLandingDataAudit]'))
+BEGIN
 ALTER TABLE [dbo].[IngestedLandingDataAudit] ADD  CONSTRAINT [DF_IngestedLandingDataAudit_ValidFrom]  DEFAULT (sysutcdatetime()) FOR [ValidFrom]
-GO
+END
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF_IngestedLandingDataAudit_ValidTo' AND parent_object_id = OBJECT_ID('[dbo].[IngestedLandingDataAudit]'))
+BEGIN
 ALTER TABLE [dbo].[IngestedLandingDataAudit] ADD  CONSTRAINT [DF_IngestedLandingDataAudit_ValidTo]  DEFAULT (CONVERT([datetime2],'9999-12-31 23:59:59.9999999')) FOR [ValidTo]
+END
+END
 GO

DeploymentComponents/azure_sql_artifacts/tables/IngestedSourceDataAudit.sql

@@ -8,6 +8,8 @@ GO
 SET QUOTED_IDENTIFIER ON
 GO
 
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[IngestedSourceDataAudit]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[IngestedSourceDataAudit](
 	[id] [int] IDENTITY(1,1) NOT NULL,
 	[source] [nvarchar](max) NULL,
@@ -33,10 +35,15 @@ WITH
 (
 SYSTEM_VERSIONING = ON (HISTORY_TABLE = [dbo].[IngestedSourceDataAuditHistory])
 )
-GO
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF_IngestedSourceDataAudit_ValidFrom' AND parent_object_id = OBJECT_ID('[dbo].[IngestedSourceDataAudit]'))
+BEGIN
 ALTER TABLE [dbo].[IngestedSourceDataAudit] ADD  CONSTRAINT [DF_IngestedSourceDataAudit_ValidFrom]  DEFAULT (sysutcdatetime()) FOR [ValidFrom]
-GO
+END
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF_IngestedSourceDataAudit_ValidTo' AND parent_object_id = OBJECT_ID('[dbo].[IngestedSourceDataAudit]'))
+BEGIN
 ALTER TABLE [dbo].[IngestedSourceDataAudit] ADD  CONSTRAINT [DF_IngestedSourceDataAudit_ValidTo]  DEFAULT (CONVERT([datetime2],'9999-12-31 23:59:59.9999999')) FOR [ValidTo]
+END
+END
 GO

DeploymentComponents/azure_sql_artifacts/tables/IngestionServiceAudit.sql

@@ -2,12 +2,15 @@
  Licensed under the MIT license. */
 
 
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[IngestionServiceAudit]') AND type in (N'U'))
+BEGIN
 CREATE TABLE dbo.IngestionServiceAudit (
     Id INT IDENTITY(1,1)
     ,LogDateTime DATETIME DEFAULT GETDATE()
     ,HttpRequest NVARCHAR(MAX)
     ,AuditMessage NVARCHAR(255)
     ,MetaDataInserted BIT
 );
+END
 GO
 

DeploymentComponents/consumers/acts/azure_sql_artifacts/consumer/tables/ControlTable.sql

@@ -1,4 +1,6 @@
-CREATE TABLE [dbo].[ControlTable] (
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[ControlTable]') AND type in (N'U'))
+BEGIN
+CREATE TABLE [dbo].[ControlTable] (
 	[id] [int] IDENTITY(1,1) NOT NULL,
     [source_id] [int] NULL,
 	[raw_filename] [nvarchar](255) NULL,
@@ -8,4 +10,5 @@
     [processed_datetime_utc] [datetime2](7) NULL
     PRIMARY KEY CLUSTERED ([Id] ASC)
 );
+END
 GO

DeploymentComponents/consumers/acts/azure_sql_artifacts/consumer/tables/IngestedLandingDataAudit.sql

@@ -2,6 +2,9 @@ SET ANSI_NULLS ON
 GO
 SET QUOTED_IDENTIFIER ON
 GO
+
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[IngestedLandingDataAudit]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[id] [int] NULL,
 	[source_filename] [nvarchar](255) NULL,
@@ -16,4 +19,5 @@ CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[control_table_record_id] int,
 	[pipeline_id] [nvarchar](255) NULL
 ) ON [PRIMARY]
+END
 GO

DeploymentComponents/consumers/beneficial_ownership/azure_sql_artifacts/consumer/tables/ControlTable.sql

@@ -2,6 +2,9 @@
 GO
 SET QUOTED_IDENTIFIER ON
 GO
+
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[ControlTable]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[ControlTable] (
 	[id] [int] IDENTITY(1,1) NOT NULL,
     [source_id] [int] NULL,
@@ -12,4 +15,5 @@ CREATE TABLE [dbo].[ControlTable] (
     [processed_datetime_utc] [datetime2](7) NULL
     PRIMARY KEY CLUSTERED ([Id] ASC)
 );
+END
 GO

DeploymentComponents/consumers/beneficial_ownership/azure_sql_artifacts/consumer/tables/DataMapping.sql

@@ -4,6 +4,8 @@ GO
 SET QUOTED_IDENTIFIER ON
 GO
 
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[DataMapping]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[DataMapping](
 	[id] [int] IDENTITY(1,1) NOT NULL,
 	[TimeStamp] [datetime2](7) NOT NULL,
@@ -33,13 +35,20 @@ CREATE TABLE [dbo].[DataMapping](
 	[id] ASC
 )WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
-GO
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF__DataMappi__activ__default' AND parent_object_id = OBJECT_ID('[dbo].[DataMapping]'))
+BEGIN
 ALTER TABLE [dbo].[DataMapping] ADD  DEFAULT (NULL) FOR [active]
-GO
+END
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF__DataMappi__activ__date1' AND parent_object_id = OBJECT_ID('[dbo].[DataMapping]'))
+BEGIN
 ALTER TABLE [dbo].[DataMapping] ADD  DEFAULT (getdate()) FOR [activeDate]
-GO
+END
 
+IF NOT EXISTS (SELECT * FROM sys.default_constraints WHERE name = 'DF__DataMappi__inact__date' AND parent_object_id = OBJECT_ID('[dbo].[DataMapping]'))
+BEGIN
 ALTER TABLE [dbo].[DataMapping] ADD  DEFAULT (getdate()) FOR [inactiveDate]
+END
+END
 GO

DeploymentComponents/consumers/beneficial_ownership/azure_sql_artifacts/consumer/tables/IngestedLandingDataAudit.sql

@@ -2,6 +2,9 @@ SET ANSI_NULLS ON
 GO
 SET QUOTED_IDENTIFIER ON
 GO
+
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[IngestedLandingDataAudit]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[id] [int] NULL,
 	[source_filename] [nvarchar](255) NULL,
@@ -16,4 +19,5 @@ CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[control_table_record_id] int,
 	[pipeline_id] [nvarchar](255) NULL
 ) ON [PRIMARY]
+END
 GO

DeploymentComponents/consumers/beneficial_ownership/azure_sql_artifacts/tables/ControlTable.sql

@@ -1,4 +1,6 @@
-CREATE TABLE [dbo].[ControlTable] (
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[ControlTable]') AND type in (N'U'))
+BEGIN
+CREATE TABLE [dbo].[ControlTable] (
 	[id] [int] IDENTITY(1,1) NOT NULL,
     [source_id] [int] NULL,
 	[raw_filename] [nvarchar](255) NULL,
@@ -8,4 +10,5 @@
     [processed_datetime_utc] [datetime2](7) NULL
     PRIMARY KEY CLUSTERED ([Id] ASC)
 );
+END
 GO

DeploymentComponents/consumers/beneficial_ownership/azure_sql_artifacts/tables/DataMapping.sql

@@ -7,6 +7,8 @@ GO
 SET QUOTED_IDENTIFIER ON
 GO
 
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[DataMapping]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[DataMapping](
 	[id] [int] IDENTITY(1,1) NOT NULL,
 	[TimeStamp] [datetime2](7) NOT NULL,
@@ -28,4 +30,5 @@ CREATE TABLE [dbo].[DataMapping](
 	[id] ASC
 )WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]
 ) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]
+END
 GO

DeploymentComponents/consumers/beneficial_ownership/azure_sql_artifacts/tables/IngestedLandingDataAudit.sql

@@ -2,6 +2,9 @@ SET ANSI_NULLS ON
 GO
 SET QUOTED_IDENTIFIER ON
 GO
+
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[IngestedLandingDataAudit]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[id] [int] NULL,
 	[source_filename] [nvarchar](255) NULL,
@@ -16,4 +19,5 @@ CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[control_table_record_id] int,
 	[pipeline_id] [nvarchar](255) NULL
 ) ON [PRIMARY]
+END
 GO

DeploymentComponents/consumers/generic/azure_sql_artifacts/consumer/tables/ControlTable.sql

@@ -1,4 +1,6 @@
-CREATE TABLE [dbo].[ControlTable] (
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[ControlTable]') AND type in (N'U'))
+BEGIN
+CREATE TABLE [dbo].[ControlTable] (
 	[id] [int] IDENTITY(1,1) NOT NULL,
     [source_id] [int] NULL,
 	[raw_filename] [nvarchar](255) NULL,
@@ -8,4 +10,5 @@
     [processed_datetime_utc] [datetime2](7) NULL
     PRIMARY KEY CLUSTERED ([Id] ASC)
 );
+END
 GO

DeploymentComponents/consumers/generic/azure_sql_artifacts/consumer/tables/IngestedLandingDataAudit.sql

@@ -2,6 +2,9 @@ SET ANSI_NULLS ON
 GO
 SET QUOTED_IDENTIFIER ON
 GO
+
+IF NOT EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[IngestedLandingDataAudit]') AND type in (N'U'))
+BEGIN
 CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[id] [int] NULL,
 	[source_filename] [nvarchar](255) NULL,
@@ -16,4 +19,5 @@ CREATE TABLE [dbo].[IngestedLandingDataAudit](
 	[control_table_record_id] int,
 	[pipeline_id] [nvarchar](255) NULL
 ) ON [PRIMARY]
+END
 GO