add new db migration code, add sqlite to postgres migration test in CI
This commit is contained in:
finn
parent
b343c02f77
commit
f87fe1338e
3 changed files with 290 additions and 48 deletions
|
|
@ -1,10 +1,11 @@
|
|||
stages:
|
||||
- build
|
||||
- test
|
||||
- publish
|
||||
|
||||
lint:
|
||||
image: golang:1.17
|
||||
stage: build
|
||||
stage: test
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install -y wget golang-go
|
||||
|
|
@ -17,6 +18,22 @@ lint:
|
|||
- diff --color=always go.sum "${CI_PROJECT_DIR}/go.sum"
|
||||
rules:
|
||||
- when: on_success
|
||||
needs: []
|
||||
|
||||
test sqlite to postgres:
|
||||
image: registry.gitlab.com/signald/signald:unstable
|
||||
stage: test
|
||||
needs:
|
||||
- "build:x86"
|
||||
before_script:
|
||||
- sudo apt-get update && sudo apt-get install -y postgresql-client-common
|
||||
script:
|
||||
- signald --migrate-data
|
||||
- echo 'CREATE DATABASE signald' | psql -h postgres -U postgres -a
|
||||
- ./signaldctl db-move postgresql://postgres@postgres/signald
|
||||
- SIGNALD_DATABASE_URI=postgresql://postgres@postgres/signald signald --migrate-data
|
||||
services:
|
||||
- postgres:latest
|
||||
|
||||
.build:
|
||||
stage: build
|
||||
|
|
|
|||
|
|
@ -17,9 +17,23 @@ import (
|
|||
"gitlab.com/signald/signald-go/cmd/signaldctl/common"
|
||||
)
|
||||
|
||||
const expectedMigrationVersion = "14"
|
||||
type Migration struct {
|
||||
InstalledRank int
|
||||
Version string
|
||||
Description string
|
||||
Script string
|
||||
Checksum int
|
||||
}
|
||||
|
||||
var (
|
||||
migrations = []Migration{
|
||||
{InstalledRank: 1, Version: "1", Description: "create tables", Script: "V1__create_tables.sql", Checksum: -1247750968},
|
||||
{InstalledRank: 2, Version: "12", Description: "create contacts table", Script: "V12__create_contacts_table.sql", Checksum: -852729911},
|
||||
{InstalledRank: 3, Version: "13", Description: "recipient registration status", Script: "V13__recipient_registration_status.sql", Checksum: 405376321},
|
||||
{InstalledRank: 4, Version: "14", Description: "multiple identity keys per account", Script: "V14__multiple_identity_keys_per_account.sql", Checksum: -1635788950},
|
||||
{InstalledRank: 5, Version: "15", Description: "profiles tables", Script: "V15__profiles_tables.sql", Checksum: 809686180},
|
||||
}
|
||||
|
||||
sqlitePath string
|
||||
postgresURL string
|
||||
MoveCmd = &cobra.Command{
|
||||
|
|
@ -79,83 +93,107 @@ var (
|
|||
log.Println("created schema")
|
||||
|
||||
if err := moveAccounts(source, dest); err != nil {
|
||||
log.Println("error migrating accounts table")
|
||||
log.Println("error moving accounts table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved accounts table")
|
||||
|
||||
if err := moveRecipients(source, dest); err != nil {
|
||||
log.Println("error migrating recipients table")
|
||||
log.Println("error moving recipients table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved recipients table")
|
||||
|
||||
if err := movePrekeys(source, dest); err != nil {
|
||||
log.Println("error migrating prekeys table")
|
||||
log.Println("error moving prekeys table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved prekeys table")
|
||||
|
||||
if err := moveSessions(source, dest); err != nil {
|
||||
log.Println("error migrating sessions table")
|
||||
log.Println("error moving sessions table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved sessions table")
|
||||
|
||||
if err := moveSignedPrekeys(source, dest); err != nil {
|
||||
log.Println("error migrating signed prekeys table")
|
||||
log.Println("error moving signed prekeys table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved signed prekeys table")
|
||||
|
||||
if err := moveIdentityKeys(source, dest); err != nil {
|
||||
log.Println("error migrating identity keys table")
|
||||
log.Println("error moving identity keys table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved identity keys table")
|
||||
|
||||
if err := moveAccountData(source, dest); err != nil {
|
||||
log.Println("error migrating account data")
|
||||
log.Println("error moving account data")
|
||||
return err
|
||||
}
|
||||
log.Println("moved account data table")
|
||||
|
||||
if err := movePendingAccountData(source, dest); err != nil {
|
||||
log.Println("error migrating pending account data tabe")
|
||||
log.Println("error moving pending account data tabe")
|
||||
return err
|
||||
}
|
||||
log.Println("moved pending account data table")
|
||||
|
||||
if err := moveSenderKeys(source, dest); err != nil {
|
||||
log.Println("error migrating sender keys table")
|
||||
log.Println("error moving sender keys table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved sender keys table")
|
||||
|
||||
if err := moveSenderKeyShared(source, dest); err != nil {
|
||||
log.Println("error migrating sender key shared table")
|
||||
log.Println("error moving sender key shared table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved sender key shared table")
|
||||
|
||||
if err := moveGroups(source, dest); err != nil {
|
||||
log.Println("error migrating groups table")
|
||||
log.Println("error moving groups table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved groups table")
|
||||
|
||||
if err := moveGroupCredentials(source, dest); err != nil {
|
||||
log.Println("error migrating group credentials table")
|
||||
log.Println("error moving group credentials table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved group credentials table")
|
||||
|
||||
if err := moveContacts(source, dest); err != nil {
|
||||
log.Println("error migrating group credentials table")
|
||||
log.Println("error moving group credentials table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved contacts table")
|
||||
|
||||
if err := moveProfileKeys(source, dest); err != nil {
|
||||
log.Println("error moving profile keys table")
|
||||
return err
|
||||
}
|
||||
log.Println("moved profile keys table")
|
||||
|
||||
if err := moveProfiles(source, dest); err != nil {
|
||||
log.Println("error moving profiles tables")
|
||||
return err
|
||||
}
|
||||
log.Println("moved profiles tables")
|
||||
|
||||
if err := moveProfileCapabilities(source, dest); err != nil {
|
||||
log.Println("error moving profile capabilities tables")
|
||||
return err
|
||||
}
|
||||
log.Println("moved profile capabilities tables")
|
||||
|
||||
if err := moveProfileBadges(source, dest); err != nil {
|
||||
log.Println("error moving profile badges tables")
|
||||
return err
|
||||
}
|
||||
log.Println("moved profile badges tables")
|
||||
|
||||
if err := os.Remove(sqlitePath); err != nil {
|
||||
log.Println("error deleting sqlite file")
|
||||
return err
|
||||
|
|
@ -175,7 +213,7 @@ func verifyMigration(source *sql.DB) error {
|
|||
defer rows.Close()
|
||||
|
||||
if !rows.Next() {
|
||||
return errors.New("source database is not up to date! Please update signald and start it to move the sqlite database to an acceptable format")
|
||||
return errors.New("source database is not up to date! Please update signald and start it to move all data into sqlite before moving data to postgres")
|
||||
}
|
||||
|
||||
var version string
|
||||
|
|
@ -184,8 +222,9 @@ func verifyMigration(source *sql.DB) error {
|
|||
return err
|
||||
}
|
||||
|
||||
expectedMigrationVersion := migrations[len(migrations)-1].Version
|
||||
if version != expectedMigrationVersion {
|
||||
return fmt.Errorf("source database must be on migration %s (found %s instead)", expectedMigrationVersion, version)
|
||||
return fmt.Errorf("source database must be on migration %s (found %s instead). Please update signald, or file an issue if the migrations are out of date", expectedMigrationVersion, version)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -197,21 +236,24 @@ func createSchema(dest *sql.DB) error {
|
|||
return err
|
||||
}
|
||||
|
||||
_, err = dest.Exec(`
|
||||
for _, migration := range migrations {
|
||||
_, err = dest.Exec(`
|
||||
INSERT INTO flyway_schema_history
|
||||
(installed_rank, version, description, type, script, checksum, installed_by, execution_time, success)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, current_user, $7, $8),
|
||||
($9, $10, $11, $12, $13, $14, current_user, $15, $16)
|
||||
VALUES ($1, $2, $3, "SQL", $4, $5, current_user, 0, true)
|
||||
`,
|
||||
// Row 1
|
||||
1, 1, "create tables", "SQL", "V1__create_tables.sql", -1247750968, 0, true,
|
||||
// Row 2
|
||||
2, 12, "create contacts table", "SQL", "V12__create_contacts_table.sql", -852729911, 0, true)
|
||||
return err
|
||||
migration.InstalledRank, migration.Version, migration.Description, migration.Script, migration.Checksum,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func moveAccounts(source *sql.DB, dest *sql.DB) error {
|
||||
rows, err := source.Query("SELECT uuid, e164, filename, server FROM accounts")
|
||||
rows, err := source.Query("SELECT uuid, e164, server FROM accounts")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -221,14 +263,13 @@ func moveAccounts(source *sql.DB, dest *sql.DB) error {
|
|||
var (
|
||||
accountUUID uuid.UUID
|
||||
e164 string
|
||||
filename string
|
||||
server uuid.UUID
|
||||
)
|
||||
err = rows.Scan(&accountUUID, &e164, &filename, &server)
|
||||
err = rows.Scan(&accountUUID, &e164, &server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dest.Exec("INSERT INTO signald_accounts (uuid, e164, filename, server) VALUES ($1, $2, $3, $4)", accountUUID, e164, filename, server)
|
||||
_, err = dest.Exec("INSERT INTO signald_accounts (uuid, e164, server) VALUES ($1, $2, $3, $4)", accountUUID, e164, server)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -603,3 +644,136 @@ func moveContacts(source *sql.DB, dest *sql.DB) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func moveProfileKeys(source *sql.DB, dest *sql.DB) error {
|
||||
rows, err := source.Query("SELECT account_uuid, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode FROM profile_keys")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
accountUUID uuid.UUID
|
||||
recipient int64
|
||||
profile_key []byte
|
||||
profile_key_credential []byte
|
||||
request_pending bool
|
||||
unidentified_access_mode int
|
||||
)
|
||||
err = rows.Scan(&accountUUID, &recipient, &profile_key, &profile_key_credential, &request_pending, &unidentified_access_mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dest.Exec(`
|
||||
INSERT INTO signald_profile_keys
|
||||
(account_uuid, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
`, accountUUID, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func moveProfiles(source *sql.DB, dest *sql.DB) error {
|
||||
rows, err := source.Query("SELECT account_uuid, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges FROM profiles")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
accountUUID uuid.UUID
|
||||
recipient int64
|
||||
last_update int64
|
||||
given_name string
|
||||
family_name string
|
||||
about string
|
||||
emoji string
|
||||
payment_address []byte
|
||||
badges string
|
||||
)
|
||||
err = rows.Scan(&accountUUID, &recipient, &last_update, &given_name, &family_name, &about, &emoji, &payment_address, &badges)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dest.Exec(`
|
||||
INSERT INTO signald_profiles
|
||||
(account_uuid, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
`, accountUUID, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func moveProfileCapabilities(source *sql.DB, dest *sql.DB) error {
|
||||
rows, err := source.Query("SELECT account_uuid, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories FROM profile_capabilities")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
accountUUID uuid.UUID
|
||||
recipient int64
|
||||
storage bool
|
||||
gv1_migration bool
|
||||
sender_key bool
|
||||
announcement_group bool
|
||||
change_number bool
|
||||
stories bool
|
||||
)
|
||||
err = rows.Scan(&accountUUID, &recipient, &storage, &gv1_migration, &sender_key, &announcement_group, &change_number, &stories)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dest.Exec(`
|
||||
INSERT INTO signald_profile_capabilities
|
||||
(account_uuid, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
`, accountUUID, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func moveProfileBadges(source *sql.DB, dest *sql.DB) error {
|
||||
rows, err := source.Query("SELECT account_uuid, id, category, name, description, sprite6 FROM profile_badges")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
accountUUID uuid.UUID
|
||||
id string
|
||||
category string
|
||||
name string
|
||||
description string
|
||||
sprite6 string
|
||||
)
|
||||
err = rows.Scan(&accountUUID, &id, &category, &name, &description, &sprite6)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = dest.Exec(`
|
||||
INSERT INTO signald_profile_badges
|
||||
(account_uuid, id, category, name, description, sprite6)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
`, accountUUID, id, category, name, description, sprite6)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,22 @@
|
|||
package db
|
||||
|
||||
var (
|
||||
// from signald in src/main/resources/db/migration/postgresql/V1__create_tables.sql
|
||||
pgScheme = `CREATE TABLE signald_message_queue (
|
||||
pgScheme = `
|
||||
CREATE TABLE public.flyway_schema_history (
|
||||
installed_rank integer NOT NULL,
|
||||
version character varying(50),
|
||||
description character varying(200) NOT NULL,
|
||||
type character varying(20) NOT NULL,
|
||||
script character varying(1000) NOT NULL,
|
||||
checksum integer,
|
||||
installed_by character varying(100) NOT NULL,
|
||||
installed_on timestamp without time zone DEFAULT now() NOT NULL,
|
||||
execution_time integer NOT NULL,
|
||||
success boolean NOT NULL
|
||||
);
|
||||
|
||||
-- from signald in src/main/resources/db/migration/postgresql/V1__create_tables.sql
|
||||
CREATE TABLE signald_message_queue (
|
||||
id SERIAL PRIMARY KEY,
|
||||
account UUID NOT NULL,
|
||||
version INTEGER NOT NULL,
|
||||
|
|
@ -93,12 +107,10 @@ var (
|
|||
CREATE TABLE signald_accounts (
|
||||
uuid UUID NOT NULL,
|
||||
e164 TEXT NOT NULL,
|
||||
filename TEXT NOT NULL,
|
||||
server UUID NOT NULL REFERENCES signald_servers(server_uuid) ON DELETE CASCADE,
|
||||
|
||||
PRIMARY KEY (uuid, e164, filename, server),
|
||||
PRIMARY KEY (uuid, e164, server),
|
||||
UNIQUE (e164),
|
||||
UNIQUE (filename),
|
||||
UNIQUE (uuid)
|
||||
);
|
||||
|
||||
|
|
@ -107,7 +119,7 @@ var (
|
|||
account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
|
||||
uuid UUID,
|
||||
e164 TEXT,
|
||||
registered BOOLEAN DEFAULT true,
|
||||
registered BOOLEAN DEFAULT true, -- from signald in src/main/resources/db/migration/postgresql/V13__recipient_registration_status.sql
|
||||
|
||||
UNIQUE (account_uuid, e164, uuid)
|
||||
);
|
||||
|
|
@ -204,19 +216,7 @@ var (
|
|||
PRIMARY KEY (account_uuid, date)
|
||||
);
|
||||
|
||||
CREATE TABLE public.flyway_schema_history (
|
||||
installed_rank integer NOT NULL,
|
||||
version character varying(50),
|
||||
description character varying(200) NOT NULL,
|
||||
type character varying(20) NOT NULL,
|
||||
script character varying(1000) NOT NULL,
|
||||
checksum integer,
|
||||
installed_by character varying(100) NOT NULL,
|
||||
installed_on timestamp without time zone DEFAULT now() NOT NULL,
|
||||
execution_time integer NOT NULL,
|
||||
success boolean NOT NULL
|
||||
);
|
||||
|
||||
-- from signald in src/main/resources/db/migration/postgresql/V12__create_contacts_table.sql
|
||||
CREATE TABLE signald_contacts (
|
||||
account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
|
||||
recipient INTEGER NOT NULL REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
|
||||
|
|
@ -228,5 +228,56 @@ var (
|
|||
|
||||
PRIMARY KEY (account_uuid, recipient)
|
||||
);
|
||||
|
||||
-- from signald in src/main/resources/db/migration/postgresql/V15__profiles_tables.sql
|
||||
|
||||
CREATE TABLE signald_profile_keys (
|
||||
account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
|
||||
recipient INTEGER NOT NULL REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
|
||||
profile_key BYTEA DEFAULT NULL,
|
||||
profile_key_credential BYTEA DEFAULT NULL,
|
||||
request_pending boolean DEFAULT FALSE,
|
||||
unidentified_access_mode int DEFAULT 0,
|
||||
|
||||
PRIMARY KEY (account_uuid, recipient)
|
||||
);
|
||||
|
||||
CREATE TABLE signald_profiles (
|
||||
account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
|
||||
recipient INTEGER NOT NULL REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
|
||||
last_update BIGINT,
|
||||
given_name TEXT,
|
||||
family_name TEXT,
|
||||
about TEXT,
|
||||
emoji TEXT,
|
||||
payment_address BYTEA,
|
||||
badges TEXT,
|
||||
|
||||
PRIMARY KEY (account_uuid, recipient)
|
||||
);
|
||||
|
||||
CREATE TABLE signald_profile_capabilities (
|
||||
account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
|
||||
recipient INTEGER NOT NULL REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
|
||||
storage BOOLEAN,
|
||||
gv1_migration BOOLEAN,
|
||||
sender_key BOOLEAN,
|
||||
announcement_group BOOLEAN,
|
||||
change_number BOOLEAN,
|
||||
stories BOOLEAN,
|
||||
|
||||
PRIMARY KEY (account_uuid, recipient)
|
||||
);
|
||||
|
||||
CREATE TABLE signald_profile_badges (
|
||||
account_uuid UUID NOT NULL REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
|
||||
id TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
sprite6 TEXT NOT NULL,
|
||||
|
||||
PRIMARY KEY (account_uuid, id)
|
||||
);
|
||||
`
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue