From f87fe1338e7c422101aa704d54e283510adc8722 Mon Sep 17 00:00:00 2001 From: finn Date: Tue, 17 May 2022 16:45:21 -0700 Subject: [PATCH] add new db migration code, add sqlite to postgres migration test in CI --- .gitlab-ci.yml | 19 ++- cmd/signaldctl/cmd/db/migrate.go | 230 ++++++++++++++++++++++++++---- cmd/signaldctl/cmd/db/postgres.go | 89 +++++++++--- 3 files changed, 290 insertions(+), 48 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9802ce6..c367862 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,10 +1,11 @@ stages: - build + - test - publish lint: image: golang:1.17 - stage: build + stage: test before_script: - apt-get update - apt-get install -y wget golang-go @@ -17,6 +18,22 @@ lint: - diff --color=always go.sum "${CI_PROJECT_DIR}/go.sum" rules: - when: on_success + needs: [] + +test sqlite to postgres: + image: registry.gitlab.com/signald/signald:unstable + stage: test + needs: + - "build:x86" + before_script: + - sudo apt-get update && sudo apt-get install -y postgresql-client-common + script: + - signald --migrate-data + - echo 'CREATE DATABASE signald' | psql -h postgres -U postgres -a + - ./signaldctl db-move postgresql://postgres@postgres/signald + - SIGNALD_DATABASE_URI=postgresql://postgres@postgres/signald signald --migrate-data + services: + - postgres:latest .build: stage: build diff --git a/cmd/signaldctl/cmd/db/migrate.go b/cmd/signaldctl/cmd/db/migrate.go index ecc8cc6..6167115 100644 --- a/cmd/signaldctl/cmd/db/migrate.go +++ b/cmd/signaldctl/cmd/db/migrate.go @@ -17,9 +17,23 @@ import ( "gitlab.com/signald/signald-go/cmd/signaldctl/common" ) -const expectedMigrationVersion = "14" +type Migration struct { + InstalledRank int + Version string + Description string + Script string + Checksum int +} var ( + migrations = []Migration{ + {InstalledRank: 1, Version: "1", Description: "create tables", Script: "V1__create_tables.sql", Checksum: -1247750968}, + {InstalledRank: 2, Version: "12", Description: "create contacts table", Script: "V12__create_contacts_table.sql", Checksum: -852729911}, + {InstalledRank: 3, Version: "13", Description: "recipient registration status", Script: "V13__recipient_registration_status.sql", Checksum: 405376321}, + {InstalledRank: 4, Version: "14", Description: "multiple identity keys per account", Script: "V14__multiple_identity_keys_per_account.sql", Checksum: -1635788950}, + {InstalledRank: 5, Version: "15", Description: "profiles tables", Script: "V15__profiles_tables.sql", Checksum: 809686180}, + } + sqlitePath string postgresURL string MoveCmd = &cobra.Command{ @@ -79,83 +93,107 @@ var ( log.Println("created schema") if err := moveAccounts(source, dest); err != nil { - log.Println("error migrating accounts table") + log.Println("error moving accounts table") return err } log.Println("moved accounts table") if err := moveRecipients(source, dest); err != nil { - log.Println("error migrating recipients table") + log.Println("error moving recipients table") return err } log.Println("moved recipients table") if err := movePrekeys(source, dest); err != nil { - log.Println("error migrating prekeys table") + log.Println("error moving prekeys table") return err } log.Println("moved prekeys table") if err := moveSessions(source, dest); err != nil { - log.Println("error migrating sessions table") + log.Println("error moving sessions table") return err } log.Println("moved sessions table") if err := moveSignedPrekeys(source, dest); err != nil { - log.Println("error migrating signed prekeys table") + log.Println("error moving signed prekeys table") return err } log.Println("moved signed prekeys table") if err := moveIdentityKeys(source, dest); err != nil { - log.Println("error migrating identity keys table") + log.Println("error moving identity keys table") return err } log.Println("moved identity keys table") if err := moveAccountData(source, dest); err != nil { - log.Println("error migrating account data") + log.Println("error moving account data") return err } log.Println("moved account data table") if err := movePendingAccountData(source, dest); err != nil { - log.Println("error migrating pending account data tabe") + log.Println("error moving pending account data tabe") return err } log.Println("moved pending account data table") if err := moveSenderKeys(source, dest); err != nil { - log.Println("error migrating sender keys table") + log.Println("error moving sender keys table") return err } log.Println("moved sender keys table") if err := moveSenderKeyShared(source, dest); err != nil { - log.Println("error migrating sender key shared table") + log.Println("error moving sender key shared table") return err } log.Println("moved sender key shared table") if err := moveGroups(source, dest); err != nil { - log.Println("error migrating groups table") + log.Println("error moving groups table") return err } log.Println("moved groups table") if err := moveGroupCredentials(source, dest); err != nil { - log.Println("error migrating group credentials table") + log.Println("error moving group credentials table") return err } log.Println("moved group credentials table") if err := moveContacts(source, dest); err != nil { - log.Println("error migrating group credentials table") + log.Println("error moving group credentials table") return err } log.Println("moved contacts table") + if err := moveProfileKeys(source, dest); err != nil { + log.Println("error moving profile keys table") + return err + } + log.Println("moved profile keys table") + + if err := moveProfiles(source, dest); err != nil { + log.Println("error moving profiles tables") + return err + } + log.Println("moved profiles tables") + + if err := moveProfileCapabilities(source, dest); err != nil { + log.Println("error moving profile capabilities tables") + return err + } + log.Println("moved profile capabilities tables") + + if err := moveProfileBadges(source, dest); err != nil { + log.Println("error moving profile badges tables") + return err + } + log.Println("moved profile badges tables") + if err := os.Remove(sqlitePath); err != nil { log.Println("error deleting sqlite file") return err @@ -175,7 +213,7 @@ func verifyMigration(source *sql.DB) error { defer rows.Close() if !rows.Next() { - return errors.New("source database is not up to date! Please update signald and start it to move the sqlite database to an acceptable format") + return errors.New("source database is not up to date! Please update signald and start it to move all data into sqlite before moving data to postgres") } var version string @@ -184,8 +222,9 @@ func verifyMigration(source *sql.DB) error { return err } + expectedMigrationVersion := migrations[len(migrations)-1].Version if version != expectedMigrationVersion { - return fmt.Errorf("source database must be on migration %s (found %s instead)", expectedMigrationVersion, version) + return fmt.Errorf("source database must be on migration %s (found %s instead). Please update signald, or file an issue if the migrations are out of date", expectedMigrationVersion, version) } return nil @@ -197,21 +236,24 @@ func createSchema(dest *sql.DB) error { return err } - _, err = dest.Exec(` + for _, migration := range migrations { + _, err = dest.Exec(` INSERT INTO flyway_schema_history (installed_rank, version, description, type, script, checksum, installed_by, execution_time, success) - VALUES ($1, $2, $3, $4, $5, $6, current_user, $7, $8), - ($9, $10, $11, $12, $13, $14, current_user, $15, $16) + VALUES ($1, $2, $3, "SQL", $4, $5, current_user, 0, true) `, - // Row 1 - 1, 1, "create tables", "SQL", "V1__create_tables.sql", -1247750968, 0, true, - // Row 2 - 2, 12, "create contacts table", "SQL", "V12__create_contacts_table.sql", -852729911, 0, true) - return err + migration.InstalledRank, migration.Version, migration.Description, migration.Script, migration.Checksum, + ) + if err != nil { + return err + } + } + + return nil } func moveAccounts(source *sql.DB, dest *sql.DB) error { - rows, err := source.Query("SELECT uuid, e164, filename, server FROM accounts") + rows, err := source.Query("SELECT uuid, e164, server FROM accounts") if err != nil { return err } @@ -221,14 +263,13 @@ func moveAccounts(source *sql.DB, dest *sql.DB) error { var ( accountUUID uuid.UUID e164 string - filename string server uuid.UUID ) - err = rows.Scan(&accountUUID, &e164, &filename, &server) + err = rows.Scan(&accountUUID, &e164, &server) if err != nil { return err } - _, err = dest.Exec("INSERT INTO signald_accounts (uuid, e164, filename, server) VALUES ($1, $2, $3, $4)", accountUUID, e164, filename, server) + _, err = dest.Exec("INSERT INTO signald_accounts (uuid, e164, server) VALUES ($1, $2, $3, $4)", accountUUID, e164, server) if err != nil { return err } @@ -603,3 +644,136 @@ func moveContacts(source *sql.DB, dest *sql.DB) error { } return nil } + +func moveProfileKeys(source *sql.DB, dest *sql.DB) error { + rows, err := source.Query("SELECT account_uuid, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode FROM profile_keys") + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var ( + accountUUID uuid.UUID + recipient int64 + profile_key []byte + profile_key_credential []byte + request_pending bool + unidentified_access_mode int + ) + err = rows.Scan(&accountUUID, &recipient, &profile_key, &profile_key_credential, &request_pending, &unidentified_access_mode) + if err != nil { + return err + } + _, err = dest.Exec(` + INSERT INTO signald_profile_keys + (account_uuid, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, accountUUID, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode) + if err != nil { + return err + } + } + return nil +} + +func moveProfiles(source *sql.DB, dest *sql.DB) error { + rows, err := source.Query("SELECT account_uuid, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges FROM profiles") + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var ( + accountUUID uuid.UUID + recipient int64 + last_update int64 + given_name string + family_name string + about string + emoji string + payment_address []byte + badges string + ) + err = rows.Scan(&accountUUID, &recipient, &last_update, &given_name, &family_name, &about, &emoji, &payment_address, &badges) + if err != nil { + return err + } + _, err = dest.Exec(` + INSERT INTO signald_profiles + (account_uuid, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + `, accountUUID, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges) + if err != nil { + return err + } + } + return nil +} + +func moveProfileCapabilities(source *sql.DB, dest *sql.DB) error { + rows, err := source.Query("SELECT account_uuid, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories FROM profile_capabilities") + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var ( + accountUUID uuid.UUID + recipient int64 + storage bool + gv1_migration bool + sender_key bool + announcement_group bool + change_number bool + stories bool + ) + err = rows.Scan(&accountUUID, &recipient, &storage, &gv1_migration, &sender_key, &announcement_group, &change_number, &stories) + if err != nil { + return err + } + _, err = dest.Exec(` + INSERT INTO signald_profile_capabilities + (account_uuid, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + `, accountUUID, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories) + if err != nil { + return err + } + } + return nil +} + +func moveProfileBadges(source *sql.DB, dest *sql.DB) error { + rows, err := source.Query("SELECT account_uuid, id, category, name, description, sprite6 FROM profile_badges") + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var ( + accountUUID uuid.UUID + id string + category string + name string + description string + sprite6 string + ) + err = rows.Scan(&accountUUID, &id, &category, &name, &description, &sprite6) + if err != nil { + return err + } + _, err = dest.Exec(` + INSERT INTO signald_profile_badges + (account_uuid, id, category, name, description, sprite6) + VALUES ($1, $2, $3, $4, $5, $6) + `, accountUUID, id, category, name, description, sprite6) + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/signaldctl/cmd/db/postgres.go b/cmd/signaldctl/cmd/db/postgres.go index 50952d1..5f7c2b3 100644 --- a/cmd/signaldctl/cmd/db/postgres.go +++ b/cmd/signaldctl/cmd/db/postgres.go @@ -1,8 +1,22 @@ package db var ( - // from signald in src/main/resources/db/migration/postgresql/V1__create_tables.sql - pgScheme = `CREATE TABLE signald_message_queue ( + pgScheme = ` + CREATE TABLE public.flyway_schema_history ( + installed_rank integer NOT NULL, + version character varying(50), + description character varying(200) NOT NULL, + type character varying(20) NOT NULL, + script character varying(1000) NOT NULL, + checksum integer, + installed_by character varying(100) NOT NULL, + installed_on timestamp without time zone DEFAULT now() NOT NULL, + execution_time integer NOT NULL, + success boolean NOT NULL + ); + + -- from signald in src/main/resources/db/migration/postgresql/V1__create_tables.sql + CREATE TABLE signald_message_queue ( id SERIAL PRIMARY KEY, account UUID NOT NULL, version INTEGER NOT NULL, @@ -93,12 +107,10 @@ var ( CREATE TABLE signald_accounts ( uuid UUID NOT NULL, e164 TEXT NOT NULL, - filename TEXT NOT NULL, server UUID NOT NULL REFERENCES signald_servers(server_uuid) ON DELETE CASCADE, - PRIMARY KEY (uuid, e164, filename, server), + PRIMARY KEY (uuid, e164, server), UNIQUE (e164), - UNIQUE (filename), UNIQUE (uuid) ); @@ -107,7 +119,7 @@ var ( account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE, uuid UUID, e164 TEXT, - registered BOOLEAN DEFAULT true, + registered BOOLEAN DEFAULT true, -- from signald in src/main/resources/db/migration/postgresql/V13__recipient_registration_status.sql UNIQUE (account_uuid, e164, uuid) ); @@ -204,19 +216,7 @@ var ( PRIMARY KEY (account_uuid, date) ); - CREATE TABLE public.flyway_schema_history ( - installed_rank integer NOT NULL, - version character varying(50), - description character varying(200) NOT NULL, - type character varying(20) NOT NULL, - script character varying(1000) NOT NULL, - checksum integer, - installed_by character varying(100) NOT NULL, - installed_on timestamp without time zone DEFAULT now() NOT NULL, - execution_time integer NOT NULL, - success boolean NOT NULL - ); - + -- from signald in src/main/resources/db/migration/postgresql/V12__create_contacts_table.sql CREATE TABLE signald_contacts ( account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE, recipient INTEGER NOT NULL REFERENCES signald_recipients(rowid) ON DELETE CASCADE, @@ -228,5 +228,56 @@ var ( PRIMARY KEY (account_uuid, recipient) ); + + -- from signald in src/main/resources/db/migration/postgresql/V15__profiles_tables.sql + + CREATE TABLE signald_profile_keys ( + account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE, + recipient INTEGER NOT NULL REFERENCES signald_recipients(rowid) ON DELETE CASCADE, + profile_key BYTEA DEFAULT NULL, + profile_key_credential BYTEA DEFAULT NULL, + request_pending boolean DEFAULT FALSE, + unidentified_access_mode int DEFAULT 0, + + PRIMARY KEY (account_uuid, recipient) + ); + + CREATE TABLE signald_profiles ( + account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE, + recipient INTEGER NOT NULL REFERENCES signald_recipients(rowid) ON DELETE CASCADE, + last_update BIGINT, + given_name TEXT, + family_name TEXT, + about TEXT, + emoji TEXT, + payment_address BYTEA, + badges TEXT, + + PRIMARY KEY (account_uuid, recipient) + ); + + CREATE TABLE signald_profile_capabilities ( + account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE, + recipient INTEGER NOT NULL REFERENCES signald_recipients(rowid) ON DELETE CASCADE, + storage BOOLEAN, + gv1_migration BOOLEAN, + sender_key BOOLEAN, + announcement_group BOOLEAN, + change_number BOOLEAN, + stories BOOLEAN, + + PRIMARY KEY (account_uuid, recipient) + ); + + CREATE TABLE signald_profile_badges ( + account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE, + id TEXT NOT NULL, + category TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + sprite6 TEXT NOT NULL, + + PRIMARY KEY (account_uuid, id) + ); ` )