internal db deployments now protected against concurrent orchestrator

instances deploying at the exact same time. No pollution of duplicate
deployments will happen.

Added SmartOrchestratorDatabaseUpdate (bool) config variable to control
use of smart deployments.

build.sh

@@ -6,7 +6,7 @@
 #
 set -e
 
-RELEASE_VERSION="1.4.447"
+RELEASE_VERSION="1.4.448"
 TOPDIR=/tmp/orchestrator-release
 export RELEASE_VERSION TOPDIR
 

go/cmd/orchestrator/main.go

@@ -717,10 +717,13 @@ Cheatsheet:
             
         reset-internal-db-deployment
             Clear internal db deployment history, use if somehow corrupted internal deployment history.
-            Orchestrator does housekeeping for its own database schema, and verifies proposed deployment vs deployment history.
-            In case of contradiction between the two orchestrator bails out. Such a contradiction is possible in the event of two
-            orchestrator instances simultaneously trying to deploy db changes (this is being worked on) 
-            By resetting history orchestrator redeploys its schema (without causing data loss).
+            When configured with '"SmartOrchestratorDatabaseUpdate": true', Orchestrator does housekeeping for its 
+            own database schema, and verifies proposed deployment vs deployment history.
+            In case of contradiction between the two orchestrator bails out. Such a contradiction should not occur, and may 
+            signify an inconsistency in the orchestrator code itself.  
+            By resetting history orchestrator redeploys its schema (without causing data loss) and accepts the new instructions
+            as the de-factor deployment rule.
+            
     `
 
 // main is the application's entry point. It will either spawn a CLI or HTTP itnerfaces.

go/config/config.go

@@ -55,6 +55,7 @@ type Configuration struct {
 	MySQLConnectTimeoutSeconds                 int      // Number of seconds before connection is aborted (driver-side)
 	DefaultInstancePort                        int      // In case port was not specified on command line
 	SkipOrchestratorDatabaseUpdate             bool     // When false, orchestrator will attempt to create & update all tables in backend database; when true, this is skipped. It makes sense to skip on command-line invocations and to enable for http or occasional invocations, or just after upgrades
+	SmartOrchestratorDatabaseUpdate            bool     // When true, orchestrator deploys internal schema based on deployment history, and only applies changes known to be uncommitted
 	SlaveLagQuery                              string   // custom query to check on slave lg (e.g. heartbeat table)
 	SlaveStartPostWaitMilliseconds             int      // Time to wait after START SLAVE before re-readong instance (give slave chance to connect to master)
 	DiscoverByShowSlaveHosts                   bool     // Attempt SHOW SLAVE HOSTS before PROCESSLIST
@@ -162,6 +163,7 @@ func NewConfiguration() *Configuration {
 		MySQLConnectTimeoutSeconds:                 5,
 		DefaultInstancePort:                        3306,
 		SkipOrchestratorDatabaseUpdate:             false,
+		SmartOrchestratorDatabaseUpdate:            false,
 		InstancePollSeconds:                        60,
 		ReadLongRunningQueries:                     true,
 		UnseenInstanceForgetHours:                  240,

go/db/db.go

@@ -33,8 +33,11 @@ var internalDBDeploymentSQL = []string{
 		  deployment_id int unsigned NOT NULL AUTO_INCREMENT,
 		  deployment_type enum('base', 'patch'),
 		  deploy_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-		  sql_statement TEXT,
-		  PRIMARY KEY (deployment_id)
+		  sql_statement TEXT CHARSET ascii NOT NULL,
+		  statement_digest VARCHAR(128) CHARSET ascii NOT NULL,
+		  statement_index INT UNSIGNED NOT NULL,
+		  PRIMARY KEY (deployment_id),
+		  UNIQUE KEY sql_index_uidx (statement_digest, statement_index)
 		) ENGINE=InnoDB DEFAULT CHARSET=ascii
 	`,
 }
@@ -618,6 +621,9 @@ func OpenOrchestrator() (*sql.DB, error) {
 
 // readInternalDeployments reads orchestrator db deployment statements that are known to have been executed
 func readInternalDeployments() (baseDeployments []string, patchDeployments []string, err error) {
+	if !config.Config.SmartOrchestratorDatabaseUpdate {
+		return baseDeployments, patchDeployments, nil
+	}
 	query := fmt.Sprintf(`
 		select 
 			deployment_type, 
@@ -659,13 +665,16 @@ func readInternalDeployments() (baseDeployments []string, patchDeployments []str
 }
 
 // writeInternalDeployment will persist a successful deployment
-func writeInternalDeployment(db *sql.DB, deploymentType string, sqlStatement string) error {
+func writeInternalDeployment(db *sql.DB, deploymentType string, sqlStatement string, statementIndex int) error {
+	if !config.Config.SmartOrchestratorDatabaseUpdate {
+		return nil
+	}
 	query := `
-        	insert into _orchestrator_db_deployment (
-				deployment_type, sql_statement) VALUES (
-				?, ?)
+        	insert ignore into _orchestrator_db_deployment (
+				deployment_type, sql_statement, statement_digest, statement_index) VALUES (
+				?, ?, CONCAT(SHA1(?), ':', LEFT(REPLACE(REPLACE(REPLACE(?, ' ', ''), '\n', ' '), '\t', ''), 60)), ?)
 				`
-	if _, err := execInternal(db, query, deploymentType, sqlStatement); err != nil {
+	if _, err := execInternal(db, query, deploymentType, sqlStatement, sqlStatement, sqlStatement, statementIndex); err != nil {
 		log.Fatalf("Unable to write to _orchestrator_db_deployment: %+v", err)
 	}
 	return nil
@@ -720,15 +729,18 @@ func deployIfNotAlreadyDeployed(db *sql.DB, queries []string, deployedQueries []
 		if queryAlreadyExecuted {
 			continue
 		}
-		log.Debugf("initOrchestratorDB executing: %.80s", strings.TrimSpace(strings.Replace(query, "\n", "", -1)))
+		if config.Config.SmartOrchestratorDatabaseUpdate {
+			log.Debugf("initOrchestratorDB executing: %.80s", strings.TrimSpace(strings.Replace(query, "\n", "", -1)))
+		}
+
 		if fatalOnError {
 			if _, err := execInternal(db, query); err != nil {
 				return log.Fatalf("Cannot initiate orchestrator: %+v", err)
 			}
 		} else {
 			execInternalSilently(db, query)
 		}
-		writeInternalDeployment(db, deploymentType, query)
+		writeInternalDeployment(db, deploymentType, query, i)
 	}
 	return nil
 }