From 01431bff8410c7043f837efcd40057280757be5d Mon Sep 17 00:00:00 2001
From: Penghao Cen <scorpiocph@gmail.com>
Date: Thu, 29 Sep 2016 22:18:28 +0800
Subject: [PATCH] verify database state with retry

---
 database/pgsql/pgsql.go | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/database/pgsql/pgsql.go b/database/pgsql/pgsql.go
index b29dad00..e26d7456 100644
--- a/database/pgsql/pgsql.go
+++ b/database/pgsql/pgsql.go
@@ -156,10 +156,26 @@ func openDatabase(registrableComponentConfig config.RegistrableComponentConfig)
 		return nil, fmt.Errorf("pgsql: could not open database: %v", err)
 	}
 
-	// Verify database state.
-	if err := pg.DB.Ping(); err != nil {
+	// Verify database state with retry.
+	ch := make(chan int, 1)
+	go func() {
+		for {
+			err := pg.DB.Ping()
+			if err == nil {
+				ch <- 1
+			} else {
+				log.Error("failed to verify database state, retry after 2 seconds")
+				time.Sleep(2 * time.Second)
+			}
+		}
+	}()
+
+	select {
+	case <-ch:
+		// verify database state ok, break
+	case <-time.After(60 * time.Second):
 		pg.Close()
-		return nil, fmt.Errorf("pgsql: could not open database: %v", err)
+		return nil, fmt.Errorf("pgsql: failed to verify database state after retry 30 times in 60 seconds: %v", err)
 	}
 
 	// Run migrations.