add new db migration code, add sqlite to postgres migration test in CI

hopefully this will keep migrations in sync

.gitlab-ci.yml

@@ -1,10 +1,11 @@
 stages:
   - build
+  - test
   - publish
 
 lint:
   image: golang:1.17
-  stage: build
+  stage: test
   before_script:
     - apt-get update
     - apt-get install -y wget golang-go
@@ -17,6 +18,30 @@ lint:
     - diff --color=always go.sum "${CI_PROJECT_DIR}/go.sum"
   rules:
     - when: on_success
+  needs: []
+
+test sqlite to postgres:
+  image:
+    name: registry.gitlab.com/signald/signald:unstable
+    entrypoint: [""]
+  stage: test
+  needs:
+    - "build:x86"
+  before_script:
+    - apt-get update && apt-get install -y postgresql-client
+  script:
+    - cd /
+    - signald --migrate-data
+    - echo 'CREATE DATABASE signald' | psql -h postgres -U postgres -a
+    - "${CI_PROJECT_DIR}/signaldctl db-move postgresql://postgres@postgres/signald?sslmode=disable"
+    - SIGNALD_DATABASE=postgresql://postgres@postgres/signald?sslmode=disable signald --migrate-data
+  variables:
+    SIGNALD_VERBOSE_LOGGING: "true"
+  services:
+    - name: postgres:latest
+      alias: postgres
+      variables:
+        POSTGRES_HOST_AUTH_METHOD: trust
 
 .build:
   stage: build

cmd/signaldctl/cmd/db/migrate.go

@@ -17,9 +17,23 @@ import (
 	"gitlab.com/signald/signald-go/cmd/signaldctl/common"
 )
 
-const expectedMigrationVersion = "14"
+type Migration struct {
+	InstalledRank int
+	Version       string
+	Description   string
+	Script        string
+	Checksum      int
+}
 
 var (
+	migrations = []Migration{
+		{InstalledRank: 1, Version: "1", Description: "create tables", Script: "V1__create_tables.sql", Checksum: -1247750968},
+		{InstalledRank: 2, Version: "12", Description: "create contacts table", Script: "V12__create_contacts_table.sql", Checksum: -852729911},
+		{InstalledRank: 3, Version: "13", Description: "recipient registration status", Script: "V13__recipient_registration_status.sql", Checksum: 405376321},
+		{InstalledRank: 4, Version: "14", Description: "multiple identity keys per account", Script: "V14__multiple_identity_keys_per_account.sql", Checksum: -1635788950},
+		{InstalledRank: 5, Version: "15", Description: "profiles tables", Script: "V15__profiles_tables.sql", Checksum: 809686180},
+	}
+
 	sqlitePath  string
 	postgresURL string
 	MoveCmd     = &cobra.Command{
@@ -79,83 +93,107 @@ var (
 			log.Println("created schema")
 
 			if err := moveAccounts(source, dest); err != nil {
-				log.Println("error migrating accounts table")
+				log.Println("error moving accounts table")
 				return err
 			}
 			log.Println("moved accounts table")
 
 			if err := moveRecipients(source, dest); err != nil {
-				log.Println("error migrating recipients table")
+				log.Println("error moving recipients table")
 				return err
 			}
 			log.Println("moved recipients table")
 
 			if err := movePrekeys(source, dest); err != nil {
-				log.Println("error migrating prekeys table")
+				log.Println("error moving prekeys table")
 				return err
 			}
 			log.Println("moved prekeys table")
 
 			if err := moveSessions(source, dest); err != nil {
-				log.Println("error migrating sessions table")
+				log.Println("error moving sessions table")
 				return err
 			}
 			log.Println("moved sessions table")
 
 			if err := moveSignedPrekeys(source, dest); err != nil {
-				log.Println("error migrating signed prekeys table")
+				log.Println("error moving signed prekeys table")
 				return err
 			}
 			log.Println("moved signed prekeys table")
 
 			if err := moveIdentityKeys(source, dest); err != nil {
-				log.Println("error migrating identity keys table")
+				log.Println("error moving identity keys table")
 				return err
 			}
 			log.Println("moved identity keys table")
 
 			if err := moveAccountData(source, dest); err != nil {
-				log.Println("error migrating account data")
+				log.Println("error moving account data")
 				return err
 			}
 			log.Println("moved account data table")
 
 			if err := movePendingAccountData(source, dest); err != nil {
-				log.Println("error migrating pending account data tabe")
+				log.Println("error moving pending account data tabe")
 				return err
 			}
 			log.Println("moved pending account data table")
 
 			if err := moveSenderKeys(source, dest); err != nil {
-				log.Println("error migrating sender keys table")
+				log.Println("error moving sender keys table")
 				return err
 			}
 			log.Println("moved sender keys table")
 
 			if err := moveSenderKeyShared(source, dest); err != nil {
-				log.Println("error migrating sender key shared table")
+				log.Println("error moving sender key shared table")
 				return err
 			}
 			log.Println("moved sender key shared table")
 
 			if err := moveGroups(source, dest); err != nil {
-				log.Println("error migrating groups table")
+				log.Println("error moving groups table")
 				return err
 			}
 			log.Println("moved groups table")
 
 			if err := moveGroupCredentials(source, dest); err != nil {
-				log.Println("error migrating group credentials table")
+				log.Println("error moving group credentials table")
 				return err
 			}
 			log.Println("moved group credentials table")
 
 			if err := moveContacts(source, dest); err != nil {
-				log.Println("error migrating group credentials table")
+				log.Println("error moving group credentials table")
 				return err
 			}
 			log.Println("moved contacts table")
 
+			if err := moveProfileKeys(source, dest); err != nil {
+				log.Println("error moving profile keys table")
+				return err
+			}
+			log.Println("moved profile keys table")
+
+			if err := moveProfiles(source, dest); err != nil {
+				log.Println("error moving profiles tables")
+				return err
+			}
+			log.Println("moved profiles tables")
+
+			if err := moveProfileCapabilities(source, dest); err != nil {
+				log.Println("error moving profile capabilities tables")
+				return err
+			}
+			log.Println("moved profile capabilities tables")
+
+			if err := moveProfileBadges(source, dest); err != nil {
+				log.Println("error moving profile badges tables")
+				return err
+			}
+			log.Println("moved profile badges tables")
+
 			if err := os.Remove(sqlitePath); err != nil {
 				log.Println("error deleting sqlite file")
 				return err
@@ -175,7 +213,7 @@ func verifyMigration(source *sql.DB) error {
 	defer rows.Close()
 
 	if !rows.Next() {
-		return errors.New("source database is not up to date! Please update signald and start it to move the sqlite database to an acceptable format")
+		return errors.New("source database is not up to date! Please update signald and start it to move all data into sqlite before moving data to postgres")
 	}
 
 	var version string
@@ -184,8 +222,9 @@ func verifyMigration(source *sql.DB) error {
 		return err
 	}
 
+	expectedMigrationVersion := migrations[len(migrations)-1].Version
 	if version != expectedMigrationVersion {
-		return fmt.Errorf("source database must be on migration %s (found %s instead)", expectedMigrationVersion, version)
+		return fmt.Errorf("source database must be on migration %s (found %s instead). Please update signald, or file an issue if the migrations are out of date", expectedMigrationVersion, version)
 	}
 
 	return nil
@@ -197,21 +236,24 @@ func createSchema(dest *sql.DB) error {
 		return err
 	}
 
-	_, err = dest.Exec(`
+	for _, migration := range migrations {
+		_, err = dest.Exec(`
         INSERT INTO flyway_schema_history
                (installed_rank, version, description, type, script, checksum, installed_by, execution_time, success)
-        VALUES ($1, $2, $3, $4, $5, $6, current_user, $7, $8),
+        VALUES ($1, $2, $3, 'SQL', $4, $5, current_user, 0, true)
-               ($9, $10, $11, $12, $13, $14, current_user, $15, $16)
         `,
-		// Row 1
+			migration.InstalledRank, migration.Version, migration.Description, migration.Script, migration.Checksum,
-		1, 1, "create tables", "SQL", "V1__create_tables.sql", -1247750968, 0, true,
+		)
-		// Row 2
+		if err != nil {
-		2, 12, "create contacts table", "SQL", "V12__create_contacts_table.sql", -852729911, 0, true)
+			return err
-	return err
+		}
+	}
+
+	return nil
 }
 
 func moveAccounts(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT uuid, e164, filename, server FROM accounts")
+	rows, err := source.Query("SELECT uuid, e164, server FROM accounts")
 	if err != nil {
 		return err
 	}
@@ -221,14 +263,13 @@ func moveAccounts(source *sql.DB, dest *sql.DB) error {
 		var (
 			accountUUID uuid.UUID
 			e164        string
-			filename    string
 			server      uuid.UUID
 		)
-		err = rows.Scan(&accountUUID, &e164, &filename, &server)
+		err = rows.Scan(&accountUUID, &e164, &server)
 		if err != nil {
 			return err
 		}
-		_, err = dest.Exec("INSERT INTO signald_accounts (uuid, e164, filename, server) VALUES ($1, $2, $3, $4)", accountUUID, e164, filename, server)
+		_, err = dest.Exec("INSERT INTO signald_accounts (uuid, e164, server) VALUES ($1, $2, $3, $4)", accountUUID, e164, server)
 		if err != nil {
 			return err
 		}
@@ -603,3 +644,136 @@ func moveContacts(source *sql.DB, dest *sql.DB) error {
 	}
 	return nil
 }
+
+func moveProfileKeys(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode FROM profile_keys")
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID              uuid.UUID
+			recipient                int64
+			profile_key              []byte
+			profile_key_credential   []byte
+			request_pending          bool
+			unidentified_access_mode int
+		)
+		err = rows.Scan(&accountUUID, &recipient, &profile_key, &profile_key_credential, &request_pending, &unidentified_access_mode)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_profile_keys
+                        (account_uuid, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode)
+                 VALUES ($1, $2, $3, $4, $5, $6, $7)
+        `, accountUUID, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func moveProfiles(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges FROM profiles")
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID     uuid.UUID
+			recipient       int64
+			last_update     int64
+			given_name      string
+			family_name     string
+			about           string
+			emoji           string
+			payment_address []byte
+			badges          string
+		)
+		err = rows.Scan(&accountUUID, &recipient, &last_update, &given_name, &family_name, &about, &emoji, &payment_address, &badges)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_profiles
+                        (account_uuid, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges)
+                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+        `, accountUUID, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func moveProfileCapabilities(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories FROM profile_capabilities")
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID        uuid.UUID
+			recipient          int64
+			storage            bool
+			gv1_migration      bool
+			sender_key         bool
+			announcement_group bool
+			change_number      bool
+			stories            bool
+		)
+		err = rows.Scan(&accountUUID, &recipient, &storage, &gv1_migration, &sender_key, &announcement_group, &change_number, &stories)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_profile_capabilities
+                        (account_uuid, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories)
+                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+        `, accountUUID, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func moveProfileBadges(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, id, category, name, description, sprite6 FROM profile_badges")
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID uuid.UUID
+			id          string
+			category    string
+			name        string
+			description string
+			sprite6     string
+		)
+		err = rows.Scan(&accountUUID, &id, &category, &name, &description, &sprite6)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_profile_badges
+                        (account_uuid, id, category, name, description, sprite6)
+                 VALUES ($1, $2, $3, $4, $5, $6)
+        `, accountUUID, id, category, name, description, sprite6)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/signaldctl/cmd/db/postgres.go

@@ -1,8 +1,22 @@
 package db
 
 var (
-	// from signald in src/main/resources/db/migration/postgresql/V1__create_tables.sql
+	pgScheme = `
-	pgScheme = `CREATE TABLE signald_message_queue (
+	CREATE TABLE public.flyway_schema_history (
+		installed_rank integer NOT NULL,
+		version character varying(50),
+		description character varying(200) NOT NULL,
+		type character varying(20) NOT NULL,
+		script character varying(1000) NOT NULL,
+		checksum integer,
+		installed_by character varying(100) NOT NULL,
+		installed_on timestamp without time zone DEFAULT now() NOT NULL,
+		execution_time integer NOT NULL,
+		success boolean NOT NULL
+	);
+
+	-- from signald in src/main/resources/db/migration/postgresql/V1__create_tables.sql
+	CREATE TABLE signald_message_queue (
 		id                          SERIAL  PRIMARY KEY,
 		account                     UUID    NOT NULL,
 		version                     INTEGER NOT NULL,
@@ -93,12 +107,10 @@ var (
 	CREATE TABLE signald_accounts (
 		uuid        UUID    NOT NULL,
 		e164        TEXT    NOT NULL,
-		filename    TEXT    NOT NULL,
 		server      UUID    NOT NULL    REFERENCES signald_servers(server_uuid) ON DELETE CASCADE,
 
-		PRIMARY KEY (uuid, e164, filename, server),
+		PRIMARY KEY (uuid, e164, server),
 		UNIQUE      (e164),
-		UNIQUE      (filename),
 		UNIQUE      (uuid)
 	);
 
@@ -107,7 +119,7 @@ var (
 		account_uuid    UUID    NOT NULL        REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
 		uuid            UUID,
 		e164            TEXT,
-		registered      BOOLEAN DEFAULT true,
+		registered      BOOLEAN DEFAULT true, -- from signald in src/main/resources/db/migration/postgresql/V13__recipient_registration_status.sql
 
 		UNIQUE (account_uuid, e164, uuid)
 	);
@@ -204,19 +216,7 @@ var (
 		PRIMARY KEY (account_uuid, date)
 	);
 
-	CREATE TABLE public.flyway_schema_history (
+	-- from signald in src/main/resources/db/migration/postgresql/V12__create_contacts_table.sql
-		installed_rank integer NOT NULL,
-		version character varying(50),
-		description character varying(200) NOT NULL,
-		type character varying(20) NOT NULL,
-		script character varying(1000) NOT NULL,
-		checksum integer,
-		installed_by character varying(100) NOT NULL,
-		installed_on timestamp without time zone DEFAULT now() NOT NULL,
-		execution_time integer NOT NULL,
-		success boolean NOT NULL
-	);
-
     CREATE TABLE signald_contacts (
         account_uuid            UUID        NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
         recipient               INTEGER     NOT NULL    REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
@@ -228,5 +228,56 @@ var (
 
         PRIMARY KEY (account_uuid, recipient)
     );
+
+	-- from signald in src/main/resources/db/migration/postgresql/V15__profiles_tables.sql
+
+	CREATE TABLE signald_profile_keys (
+		account_uuid             UUID       NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
+		recipient                INTEGER    NOT NULL    REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
+		profile_key              BYTEA      DEFAULT NULL,
+		profile_key_credential   BYTEA      DEFAULT NULL,
+		request_pending          boolean    DEFAULT FALSE,
+		unidentified_access_mode int        DEFAULT 0,
+
+		PRIMARY KEY (account_uuid, recipient)
+	);
+
+	CREATE TABLE signald_profiles (
+		account_uuid             UUID       NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
+		recipient                INTEGER    NOT NULL    REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
+		last_update              BIGINT,
+		given_name               TEXT,
+		family_name              TEXT,
+		about                    TEXT,
+		emoji                    TEXT,
+		payment_address          BYTEA,
+		badges                   TEXT,
+
+		PRIMARY KEY (account_uuid, recipient)
+	);
+
+	CREATE TABLE signald_profile_capabilities (
+		account_uuid        UUID       NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
+		recipient           INTEGER    NOT NULL    REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
+		storage             BOOLEAN,
+		gv1_migration       BOOLEAN,
+		sender_key          BOOLEAN,
+		announcement_group  BOOLEAN,
+		change_number       BOOLEAN,
+		stories             BOOLEAN,
+
+		PRIMARY KEY (account_uuid, recipient)
+	);
+
+	CREATE TABLE signald_profile_badges (
+		account_uuid  UUID       NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
+		id            TEXT       NOT NULL,
+		category      TEXT       NOT NULL,
+		name          TEXT       NOT NULL,
+		description   TEXT       NOT NULL,
+		sprite6       TEXT       NOT NULL,
+
+		PRIMARY KEY (account_uuid, id)
+	);
 `
 )