improve error handling for listen

Fixes #15

.gitlab-ci.yml

@@ -1,10 +1,11 @@
 stages:
   - build
+  - test
   - publish
 
 lint:
-  image: golang:latest
+  image: golang:1.17
-  stage: build
+  stage: test
   before_script:
     - apt-get update
     - apt-get install -y wget golang-go
@@ -17,6 +18,30 @@ lint:
     - diff --color=always go.sum "${CI_PROJECT_DIR}/go.sum"
   rules:
     - when: on_success
+  needs: []
+
+test sqlite to postgres:
+  image:
+    name: registry.gitlab.com/signald/signald:unstable
+    entrypoint: [""]
+  stage: test
+  needs:
+    - "build:x86"
+  before_script:
+    - apt-get update && apt-get install -y postgresql-client
+  script:
+    - cd /
+    - signald --migrate-data
+    - echo 'CREATE DATABASE signald' | psql -h postgres -U postgres -a
+    - "${CI_PROJECT_DIR}/signaldctl db-move postgresql://postgres@postgres/signald?sslmode=disable"
+    - SIGNALD_DATABASE=postgresql://postgres@postgres/signald?sslmode=disable signald --migrate-data
+  variables:
+    SIGNALD_VERBOSE_LOGGING: "true"
+  services:
+    - name: postgres:latest
+      alias: postgres
+      variables:
+        POSTGRES_HOST_AUTH_METHOD: trust
 
 .build:
   stage: build
@@ -37,27 +62,22 @@ lint:
 .build-deb:
   stage: build
   image: debian:buster
-  script:
+  before_script:
     - echo deb http://deb.debian.org/debian buster-backports main > /etc/apt/sources.list.d/backports.list
     - apt-get update
-    - apt-get install -y -t buster-backports git-buildpackage dh-golang bash-completion golang-any golang-github-spf13-cobra-dev golang-github-spf13-viper-dev golang-github-google-uuid-dev golang-github-mattn-go-sqlite3-dev golang-github-lib-pq-dev golang-github-satori-go.uuid-dev
+    - apt-get install -y -t buster-backports git-buildpackage dh-golang bash-completion golang-any golang-github-spf13-cobra-dev golang-github-spf13-viper-dev golang-github-google-uuid-dev golang-github-mattn-go-sqlite3-dev golang-github-lib-pq-dev golang-github-satori-go.uuid-dev wget unzip
+    - wget -O golang-github-mdp-qrterminal.zip --quiet "https://gitlab.com/api/v4/projects/signald%2Flibraries%2Fgolang-github-mdp-qrterminal/jobs/artifacts/master/download?job=build"
+    - wget -O golang-github-jedib0t-go-pretty.zip --quiet "https://gitlab.com/api/v4/projects/signald%2Flibraries%2Fgolang-github-jedib0t-go-pretty/jobs/artifacts/master/download?job=build"
+    - for z in *.zip; do unzip $z; done
     - apt-get install -y ./*.deb && rm -vf *.deb
+  script:
     - 'sed -i "s/^Architecture:.*/Architecture: ${ARCH}/g" debian/control'
     - go run ./cmd/signaldctl doc -o man
     - go run ./cmd/signaldctl completion bash > debian/package.bash-completion
     - ls *.1 > debian/manpages
-    - gbp dch --ignore-branch --debian-tag="%(version)s" --git-author --new-version="$(./version.sh)"
+    - gbp dch --ignore-branch --debian-tag="%(version)s" --git-author --new-version="$(./version.sh | cut -c2-)"
     - dpkg-buildpackage -us -uc -b
     - mv ../*.deb .
-  needs:
-  - project: signald/libraries/golang-github-mdp-qrterminal
-    job: build
-    ref: master
-    artifacts: true
-  - project: signald/libraries/golang-github-jedib0t-go-pretty
-    job: build
-    ref: master
-    artifacts: true
   variables:
     SIGNALDCTL_PUBLIC_DOC_MODE: "on"
   artifacts:

cmd/signaldctl/cmd/account/link/link-account.go

@@ -34,6 +34,7 @@ import (
 var (
 	testing    bool
 	deviceName string
+	overwrite  bool
 
 	LinkAccountCmd = &cobra.Command{
 		Use:   "link",
@@ -80,6 +81,7 @@ var (
 			finishReq := v1.FinishLinkRequest{
 				DeviceName: deviceName,
 				SessionId:  response.SessionId,
+				Overwrite:  overwrite,
 			}
 
 			_, err = finishReq.Submit(common.Signald)
@@ -99,4 +101,5 @@ func init() {
 	}
 	LinkAccountCmd.Flags().BoolVarP(&testing, "testing", "t", false, "use the Signal testing server")
 	LinkAccountCmd.Flags().StringVarP(&deviceName, "device-name", "n", name, "the name of this device. shown to other devices on the signal account")
+	LinkAccountCmd.Flags().BoolVar(&overwrite, "overwrite", false, "if an account with the same id already exists in signald's database, delete it before linking")
 }

cmd/signaldctl/cmd/account/requestsync/main.go

@@ -0,0 +1,36 @@
+package requestsync
+
+import (
+	"log"
+
+	"github.com/spf13/cobra"
+	"gitlab.com/signald/signald-go/cmd/signaldctl/common"
+	"gitlab.com/signald/signald-go/cmd/signaldctl/config"
+	v1 "gitlab.com/signald/signald-go/signald/client-protocol/v1"
+)
+
+var (
+	account        string
+	RequestSyncCmd = &cobra.Command{
+		Use:   "request-sync",
+		Short: "Ask other devices on the account to send sync data. Must subscribe for result",
+		PreRun: func(cmd *cobra.Command, args []string) {
+			if account == "" {
+				account = config.Config.DefaultAccount
+			}
+		},
+		Run: func(_ *cobra.Command, _ []string) {
+			go common.Signald.Listen(nil)
+			req := &v1.RequestSyncRequest{Account: account}
+			err := req.Submit(common.Signald)
+			if err != nil {
+				panic(err)
+			}
+			log.Println("sync requested. Must be subscribed to receive response")
+		},
+	}
+)
+
+func init() {
+	RequestSyncCmd.Flags().StringVarP(&account, "account", "a", "", "the signald account to use")
+}

cmd/signaldctl/cmd/account/root.go

@@ -23,6 +23,7 @@ import (
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/account/list"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/account/register"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/account/remoteconfig"
+	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/account/requestsync"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/account/setprofile"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/account/verify"
 )
@@ -37,7 +38,8 @@ func init() {
 	AccountCmd.AddCommand(link.LinkAccountCmd)
 	AccountCmd.AddCommand(list.ListAccountCmd)
 	AccountCmd.AddCommand(register.RegisterAccountCmd)
-	AccountCmd.AddCommand(verify.VerifyAccountCmd)
-	AccountCmd.AddCommand(setprofile.SetProfileCmd)
 	AccountCmd.AddCommand(remoteconfig.RemoteConfigCmd)
+	AccountCmd.AddCommand(requestsync.RequestSyncCmd)
+	AccountCmd.AddCommand(setprofile.SetProfileCmd)
+	AccountCmd.AddCommand(verify.VerifyAccountCmd)
 }

cmd/signaldctl/cmd/account/setprofile/set-profile.go

@@ -27,10 +27,14 @@ import (
 
 var (
 	account string
+	name    string
+	avatar  string
+	emoji   string
+	about   string
 
 	SetProfileCmd = &cobra.Command{
-		Use:   "set-profile name",
+		Use:   "set-profile [name]",
-		Short: "updates the profile data with a new name",
+		Short: "update an account's profile data",
 		PreRun: func(cmd *cobra.Command, args []string) {
 			if account == "" {
 				account = config.Config.DefaultAccount
@@ -39,16 +43,19 @@ var (
 				common.Must(cmd.Help())
 				log.Fatal("No account specified. Please specify with --account or set a default")
 			}
-			if len(args) != 1 {
+
-				common.Must(cmd.Help())
+			if len(args) > 0 {
-				log.Fatal("must specify a name")
+				name = args[0]
 			}
 		},
-		Run: func(_ *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
 			go common.Signald.Listen(nil)
 			req := v1.SetProfile{
 				Account:    account,
-				Name:    args[0],
+				Name:       name,
+				AvatarFile: avatar,
+				Emoji:      emoji,
+				About:      about,
 			}
 			err := req.Submit(common.Signald)
 			if err != nil {
@@ -61,4 +68,7 @@ var (
 
 func init() {
 	SetProfileCmd.Flags().StringVarP(&account, "account", "a", "", "the signald account to use")
+	SetProfileCmd.Flags().StringVarP(&avatar, "avatar", "A", "", "path to avatar file")
+	SetProfileCmd.Flags().StringVar(&emoji, "emoji", "", "an emoji to be shown next to the about section")
+	SetProfileCmd.Flags().StringVar(&about, "about", "", "profile about section")
 }

cmd/signaldctl/cmd/db/migrate.go

@@ -17,7 +17,25 @@ import (
 	"gitlab.com/signald/signald-go/cmd/signaldctl/common"
 )
 
+type Migration struct {
+	InstalledRank int
+	Version       string
+	Description   string
+	Script        string
+	Checksum      int
+}
+
 var (
+	migrations = []Migration{
+		{InstalledRank: 1, Version: "1", Description: "create tables", Script: "V1__create_tables.sql", Checksum: -1247750968},
+		{InstalledRank: 2, Version: "12", Description: "create contacts table", Script: "V12__create_contacts_table.sql", Checksum: -852729911},
+		{InstalledRank: 3, Version: "13", Description: "recipient registration status", Script: "V13__recipient_registration_status.sql", Checksum: 405376321},
+		{InstalledRank: 4, Version: "14", Description: "multiple identity keys per account", Script: "V14__multiple_identity_keys_per_account.sql", Checksum: -1635788950},
+		{InstalledRank: 5, Version: "15", Description: "profiles tables", Script: "V15__profiles_tables.sql", Checksum: 809686180},
+		{InstalledRank: 6, Version: "16", Description: "destination uuid in envelope", Script: "V16__destination_uuid_in_envelope.sql", Checksum: 357656854},
+		{InstalledRank: 7, Version: "17", Description: "update server ca", Script: "V17__update_server_ca.sql", Checksum: 1647934070},
+	}
+
 	sqlitePath  string
 	postgresURL string
 	MoveCmd     = &cobra.Command{
@@ -43,7 +61,7 @@ var (
 			}
 			return nil
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			source, err := sql.Open("sqlite3", sqlitePath)
 			if err != nil {
 				return err
@@ -76,77 +94,37 @@ var (
 			}
 			log.Println("created schema")
 
-			if err := moveAccounts(source, dest); err != nil {
+			migrate := func(fn func(*sql.DB, *sql.DB) error, targetName string) {
-				log.Println("error migrating accounts table")
+				if err = fn(source, dest); err != nil {
-				return err
+					log.Println("error moving", targetName)
+					panic(err)
 				}
-			log.Println("moved accounts table")
+				log.Println("moved", targetName)
+			}
+			defer func() {
+				if r := recover(); r != nil && r != err {
+					// If r is something other than the error returned via the named return, re-panic it
+					panic(r)
+				}
+			}()
 
-			if err := moveRecipients(source, dest); err != nil {
+			migrate(moveAccounts, "accounts table")
-				log.Println("error migrating recipients table")
+			migrate(moveRecipients, "recipients table")
-				return err
+			migrate(movePrekeys, "prekeys table")
-			}
+			migrate(moveSessions, "sessions table")
-			log.Println("moved recipients table")
+			migrate(moveSignedPrekeys, "signed prekeys table")
-
+			migrate(moveIdentityKeys, "identity keys table")
-			if err := movePrekeys(source, dest); err != nil {
+			migrate(moveAccountData, "account data")
-				log.Println("error migrating prekeys table")
+			migrate(movePendingAccountData, "pending account data table")
-				return err
+			migrate(moveSenderKeys, "sender keys table")
-			}
+			migrate(moveSenderKeyShared, "sender key shared table")
-			log.Println("moved prekeys table")
+			migrate(moveGroups, "groups table")
-
+			migrate(moveGroupCredentials, "group credentials table")
-			if err := moveSessions(source, dest); err != nil {
+			migrate(moveContacts, "contacts table")
-				log.Println("error migrating sessions table")
+			migrate(moveProfileKeys, "profile keys table")
-				return err
+			migrate(moveProfiles, "profiles tables")
-			}
+			migrate(moveProfileCapabilities, "profile capabilities tables")
-			log.Println("moved sessions table")
+			migrate(moveProfileBadges, "profile badges tables")
-
-			if err := moveSignedPrekeys(source, dest); err != nil {
-				log.Println("error migrating signed prekeys table")
-				return err
-			}
-			log.Println("moved signed prekeys table")
-
-			if err := moveIdentityKeys(source, dest); err != nil {
-				log.Println("error migrating identity keys table")
-				return err
-			}
-			log.Println("moved identity keys table")
-
-			if err := moveAccountData(source, dest); err != nil {
-				log.Println("error migrating account data")
-				return err
-			}
-			log.Println("moved account data table")
-
-			if err := movePendingAccountData(source, dest); err != nil {
-				log.Println("error migrating pending account data tabe")
-				return err
-			}
-			log.Println("moved pending account data table")
-
-			if err := moveSenderKeys(source, dest); err != nil {
-				log.Println("error migrating sender keys table")
-				return err
-			}
-			log.Println("moved sender keys table")
-
-			if err := moveSenderKeyShared(source, dest); err != nil {
-				log.Println("error migrating sender key shared table")
-				return err
-			}
-			log.Println("moved sender key shared table")
-
-			if err := moveGroups(source, dest); err != nil {
-				log.Println("error migrating groups table")
-				return err
-			}
-			log.Println("moved groups table")
-
-			if err := moveGroupCredentials(source, dest); err != nil {
-				log.Println("error migrating group credentials table")
-				return err
-			}
-			log.Println("moved group credentials table")
 
 			if err := os.Remove(sqlitePath); err != nil {
 				log.Println("error deleting sqlite file")
@@ -159,15 +137,28 @@ var (
 )
 
 func verifyMigration(source *sql.DB) error {
-	rows, err := source.Query("SELECT * FROM flyway_schema_history WHERE version = 11")
+	// Lower bound of the database state.
+	rows, err := source.Query("SELECT version FROM flyway_schema_history ORDER BY installed_rank DESC LIMIT 1")
 	if err != nil {
 		return err
 	}
 	defer rows.Close()
 
 	if !rows.Next() {
-		return errors.New("source database is not up to date! Please update to signald 0.17.0-16-5101c6ef and start it to move the sqlite database to an acceptable format")
+		return errors.New("source database is not up to date! Please update signald and start it to move all data into sqlite before moving data to postgres")
 	}
+
+	var version string
+	err = rows.Scan(&version)
+	if err != nil {
+		return err
+	}
+
+	expectedMigrationVersion := migrations[len(migrations)-1].Version
+	if version != expectedMigrationVersion {
+		return fmt.Errorf("source database must be on migration %s (found %s instead). Please update signald, or file an issue if the migrations are out of date", expectedMigrationVersion, version)
+	}
+
 	return nil
 }
 
@@ -177,13 +168,24 @@ func createSchema(dest *sql.DB) error {
 		return err
 	}
 
-	_, err = dest.Exec("INSERT INTO flyway_schema_history (installed_rank, version, description, type, script, checksum, installed_by, execution_time, success) VALUES ($1, $2, $3, $4, $5, $6, current_user, $7, $8)",
+	for _, migration := range migrations {
-		1, 1, "create tables", "SQL", "V1__create_tables.sql", -1247750968, 0, true)
+		_, err = dest.Exec(`
+        INSERT INTO flyway_schema_history
+               (installed_rank, version, description, type, script, checksum, installed_by, execution_time, success)
+        VALUES ($1, $2, $3, 'SQL', $4, $5, current_user, 0, true)
+        `,
+			migration.InstalledRank, migration.Version, migration.Description, migration.Script, migration.Checksum,
+		)
+		if err != nil {
 			return err
 		}
+	}
+
+	return nil
+}
 
 func moveAccounts(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT uuid, e164, filename, server FROM accounts")
+	rows, err := source.Query("SELECT uuid, e164, server FROM accounts")
 	if err != nil {
 		return err
 	}
@@ -193,14 +195,13 @@ func moveAccounts(source *sql.DB, dest *sql.DB) error {
 		var (
 			accountUUID uuid.UUID
 			e164        string
-			filename    string
 			server      uuid.UUID
 		)
-		err = rows.Scan(&accountUUID, &e164, &filename, &server)
+		err = rows.Scan(&accountUUID, &e164, &server)
 		if err != nil {
 			return err
 		}
-		_, err = dest.Exec("INSERT INTO signald_accounts (uuid, e164, filename, server) VALUES ($1, $2, $3, $4)", accountUUID, e164, filename, server)
+		_, err = dest.Exec("INSERT INTO signald_accounts (uuid, e164, server) VALUES ($1, $2, $3)", accountUUID, e164, server)
 		if err != nil {
 			return err
 		}
@@ -209,7 +210,9 @@ func moveAccounts(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveRecipients(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT rowid, account_uuid, uuid, e164 FROM recipients")
+	rows, err := source.Query("SELECT rowid, account_uuid, uuid, e164, registered FROM recipients" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
 	if err != nil {
 		return err
 	}
@@ -221,8 +224,9 @@ func moveRecipients(source *sql.DB, dest *sql.DB) error {
 			accountUUID   uuid.UUID
 			recipientUUID uuid.NullUUID
 			e164          sql.NullString
+			registered    bool
 		)
-		err = rows.Scan(&rowID, &accountUUID, &recipientUUID, &e164)
+		err = rows.Scan(&rowID, &accountUUID, &recipientUUID, &e164, &registered)
 		if err != nil {
 			return err
 		}
@@ -233,7 +237,7 @@ func moveRecipients(source *sql.DB, dest *sql.DB) error {
 			e164.String = ""
 		}
 
-		_, err = dest.Exec("INSERT INTO signald_recipients (rowid, account_uuid, uuid, e164) VALUES ($1, $2, $3, $4)", rowID, accountUUID, recipientUUID, e164)
+		_, err = dest.Exec("INSERT INTO signald_recipients (rowid, account_uuid, uuid, e164, registered) VALUES ($1, $2, $3, $4, $5)", rowID, accountUUID, recipientUUID, e164, registered)
 		if err != nil {
 			return err
 		}
@@ -248,7 +252,9 @@ func moveRecipients(source *sql.DB, dest *sql.DB) error {
 }
 
 func movePrekeys(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT account_uuid, id, record FROM prekeys")
+	rows, err := source.Query("SELECT account_uuid, id, record FROM prekeys" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
 	if err != nil {
 		return err
 	}
@@ -273,7 +279,10 @@ func movePrekeys(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveSessions(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT account_uuid, recipient, device_id, record FROM sessions")
+	rows, err := source.Query("SELECT account_uuid, recipient, device_id, record FROM sessions" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)" +
+		" AND   recipient IN (SELECT DISTINCT rowid FROM recipients)",
+	)
 	if err != nil {
 		return err
 	}
@@ -308,7 +317,9 @@ func moveSessions(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveSignedPrekeys(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT account_uuid, id, record FROM signed_prekeys")
+	rows, err := source.Query("SELECT account_uuid, id, record FROM signed_prekeys" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
 	if err != nil {
 		return err
 	}
@@ -333,7 +344,9 @@ func moveSignedPrekeys(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveIdentityKeys(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT account_uuid, recipient, identity_key, trust_level, added FROM identity_keys")
+	rows, err := source.Query("SELECT account_uuid, recipient, identity_key, trust_level, added FROM identity_keys" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
 	if err != nil {
 		return err
 	}
@@ -360,7 +373,9 @@ func moveIdentityKeys(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveAccountData(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT account_uuid, key, value FROM account_data")
+	rows, err := source.Query("SELECT account_uuid, key, value FROM account_data" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
 	if err != nil {
 		return err
 	}
@@ -410,7 +425,9 @@ func movePendingAccountData(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveSenderKeys(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT account_uuid, address, device, distribution_id, record, created_at FROM sender_keys")
+	rows, err := source.Query("SELECT account_uuid, address, device, distribution_id, record, created_at FROM sender_keys" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
 	if err != nil {
 		return err
 	}
@@ -446,7 +463,9 @@ func moveSenderKeys(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveSenderKeyShared(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT account_uuid, distribution_id, address, device FROM sender_key_shared")
+	rows, err := source.Query("SELECT account_uuid, distribution_id, address, device FROM sender_key_shared" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
 	if err != nil {
 		return err
 	}
@@ -482,7 +501,9 @@ func moveSenderKeyShared(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveGroups(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT rowid, account_uuid, group_id, master_key, revision, last_avatar_fetch, distribution_id, group_info FROM groups")
+	rows, err := source.Query("SELECT rowid, account_uuid, group_id, master_key, revision, last_avatar_fetch, distribution_id, group_info FROM groups" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
 	if err != nil {
 		return err
 	}
@@ -518,7 +539,10 @@ func moveGroups(source *sql.DB, dest *sql.DB) error {
 }
 
 func moveGroupCredentials(source *sql.DB, dest *sql.DB) error {
-	rows, err := source.Query("SELECT account_uuid, date, credential FROM group_credentials")
+	rows, err := source.Query("SELECT account_uuid, date, max(credential) FROM group_credentials" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)" +
+		" GROUP BY account_uuid, date",
+	)
 	if err != nil {
 		return err
 	}
@@ -541,3 +565,183 @@ func moveGroupCredentials(source *sql.DB, dest *sql.DB) error {
 	}
 	return nil
 }
+
+func moveContacts(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, recipient, name, color, profile_key, message_expiration_time, inbox_position FROM contacts" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)" +
+		" AND   recipient IN (SELECT DISTINCT rowid FROM recipients)",
+	)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID             uuid.UUID
+			recipient               int64
+			name                    sql.NullString
+			color                   sql.NullString
+			profile_key             []byte
+			message_expiration_time sql.NullInt64
+			inbox_position          sql.NullInt64
+		)
+		err = rows.Scan(&accountUUID, &recipient, &name, &color, &profile_key, &message_expiration_time, &inbox_position)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_contacts
+                        (account_uuid, recipient, name, color, profile_key, message_expiration_time, inbox_position)
+                 VALUES ($1, $2, $3, $4, $5, $6, $7)
+        `, accountUUID, recipient, name, color, profile_key, message_expiration_time, inbox_position)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func moveProfileKeys(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode FROM profile_keys" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)" +
+		" AND   recipient IN (SELECT DISTINCT rowid FROM recipients)",
+	)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID              uuid.UUID
+			recipient                int64
+			profile_key              []byte
+			profile_key_credential   []byte
+			request_pending          bool
+			unidentified_access_mode int
+		)
+		err = rows.Scan(&accountUUID, &recipient, &profile_key, &profile_key_credential, &request_pending, &unidentified_access_mode)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_profile_keys
+                        (account_uuid, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode)
+                 VALUES ($1, $2, $3, $4, $5, $6)
+        `, accountUUID, recipient, profile_key, profile_key_credential, request_pending, unidentified_access_mode)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func moveProfiles(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges FROM profiles" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)" +
+		" AND   recipient IN (SELECT DISTINCT rowid FROM recipients)",
+	)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID     uuid.UUID
+			recipient       int64
+			last_update     int64
+			given_name      string
+			family_name     string
+			about           string
+			emoji           string
+			payment_address []byte
+			badges          sql.NullString
+		)
+		err = rows.Scan(&accountUUID, &recipient, &last_update, &given_name, &family_name, &about, &emoji, &payment_address, &badges)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_profiles
+                        (account_uuid, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges)
+                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+        `, accountUUID, recipient, last_update, given_name, family_name, about, emoji, payment_address, badges)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func moveProfileCapabilities(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories FROM profile_capabilities" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)" +
+		" AND   recipient IN (SELECT DISTINCT rowid FROM recipients)",
+	)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID        uuid.UUID
+			recipient          int64
+			storage            bool
+			gv1_migration      bool
+			sender_key         bool
+			announcement_group bool
+			change_number      bool
+			stories            bool
+		)
+		err = rows.Scan(&accountUUID, &recipient, &storage, &gv1_migration, &sender_key, &announcement_group, &change_number, &stories)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_profile_capabilities
+                        (account_uuid, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories)
+                 VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+        `, accountUUID, recipient, storage, gv1_migration, sender_key, announcement_group, change_number, stories)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func moveProfileBadges(source *sql.DB, dest *sql.DB) error {
+	rows, err := source.Query("SELECT account_uuid, id, category, name, description, sprite6 FROM profile_badges" +
+		" WHERE account_uuid IN (SELECT DISTINCT uuid FROM accounts)",
+	)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var (
+			accountUUID uuid.UUID
+			id          string
+			category    string
+			name        string
+			description string
+			sprite6     string
+		)
+		err = rows.Scan(&accountUUID, &id, &category, &name, &description, &sprite6)
+		if err != nil {
+			return err
+		}
+		_, err = dest.Exec(`
+            INSERT INTO signald_profile_badges
+                        (account_uuid, id, category, name, description, sprite6)
+                 VALUES ($1, $2, $3, $4, $5, $6)
+        `, accountUUID, id, category, name, description, sprite6)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/signaldctl/cmd/db/postgres.go

@@ -1,8 +1,22 @@
 package db
 
 var (
-	// from signald in src/main/resources/db/migration/postgresql/V1__create_tables.sql
+	pgScheme = `
-	pgScheme = `CREATE TABLE signald_message_queue (
+	CREATE TABLE public.flyway_schema_history (
+		installed_rank integer NOT NULL,
+		version character varying(50),
+		description character varying(200) NOT NULL,
+		type character varying(20) NOT NULL,
+		script character varying(1000) NOT NULL,
+		checksum integer,
+		installed_by character varying(100) NOT NULL,
+		installed_on timestamp without time zone DEFAULT now() NOT NULL,
+		execution_time integer NOT NULL,
+		success boolean NOT NULL
+	);
+
+	-- from signald in src/main/resources/db/migration/postgresql/V1__create_tables.sql
+	CREATE TABLE signald_message_queue (
 		id                          SERIAL  PRIMARY KEY,
 		account                     UUID    NOT NULL,
 		version                     INTEGER NOT NULL,
@@ -15,7 +29,8 @@ var (
 		legacy_message              BYTEA,
 		server_received_timestamp   BIGINT,
 		server_delivered_timestamp  BIGINT,
-		server_uuid                 UUID
+		server_uuid                 UUID,
+		destination_uuid            TEXT
 	);
 
 	CREATE TABLE signald_servers (
@@ -93,12 +108,10 @@ var (
 	CREATE TABLE signald_accounts (
 		uuid        UUID    NOT NULL,
 		e164        TEXT    NOT NULL,
-		filename    TEXT    NOT NULL,
 		server      UUID    NOT NULL    REFERENCES signald_servers(server_uuid) ON DELETE CASCADE,
 
-		PRIMARY KEY (uuid, e164, filename, server),
+		PRIMARY KEY (uuid, e164, server),
 		UNIQUE      (e164),
-		UNIQUE      (filename),
 		UNIQUE      (uuid)
 	);
 
@@ -107,6 +120,7 @@ var (
 		account_uuid    UUID    NOT NULL        REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
 		uuid            UUID,
 		e164            TEXT,
+		registered      BOOLEAN DEFAULT true, -- from signald in src/main/resources/db/migration/postgresql/V13__recipient_registration_status.sql
 
 		UNIQUE (account_uuid, e164, uuid)
 	);
@@ -203,17 +217,68 @@ var (
 		PRIMARY KEY (account_uuid, date)
 	);
 
-	CREATE TABLE public.flyway_schema_history (
+	-- from signald in src/main/resources/db/migration/postgresql/V12__create_contacts_table.sql
-		installed_rank integer NOT NULL,
+    CREATE TABLE signald_contacts (
-		version character varying(50),
+        account_uuid            UUID        NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
-		description character varying(200) NOT NULL,
+        recipient               INTEGER     NOT NULL    REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
-		type character varying(20) NOT NULL,
+        name                    TEXT,
-		script character varying(1000) NOT NULL,
+        color                   TEXT,
-		checksum integer,
+        profile_key             BYTEA,
-		installed_by character varying(100) NOT NULL,
+        message_expiration_time INTEGER,
-		installed_on timestamp without time zone DEFAULT now() NOT NULL,
+        inbox_position          INTEGER,
-		execution_time integer NOT NULL,
+
-		success boolean NOT NULL
+        PRIMARY KEY (account_uuid, recipient)
+    );
+
+	-- from signald in src/main/resources/db/migration/postgresql/V15__profiles_tables.sql
+
+	CREATE TABLE signald_profile_keys (
+		account_uuid             UUID       NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
+		recipient                INTEGER    NOT NULL    REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
+		profile_key              BYTEA      DEFAULT NULL,
+		profile_key_credential   BYTEA      DEFAULT NULL,
+		request_pending          boolean    DEFAULT FALSE,
+		unidentified_access_mode int        DEFAULT 0,
+
+		PRIMARY KEY (account_uuid, recipient)
+	);
+
+	CREATE TABLE signald_profiles (
+		account_uuid             UUID       NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
+		recipient                INTEGER    NOT NULL    REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
+		last_update              BIGINT,
+		given_name               TEXT,
+		family_name              TEXT,
+		about                    TEXT,
+		emoji                    TEXT,
+		payment_address          BYTEA,
+		badges                   TEXT,
+
+		PRIMARY KEY (account_uuid, recipient)
+	);
+
+	CREATE TABLE signald_profile_capabilities (
+		account_uuid        UUID       NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
+		recipient           INTEGER    NOT NULL    REFERENCES signald_recipients(rowid) ON DELETE CASCADE,
+		storage             BOOLEAN,
+		gv1_migration       BOOLEAN,
+		sender_key          BOOLEAN,
+		announcement_group  BOOLEAN,
+		change_number       BOOLEAN,
+		stories             BOOLEAN,
+
+		PRIMARY KEY (account_uuid, recipient)
+	);
+
+	CREATE TABLE signald_profile_badges (
+		account_uuid  UUID       NOT NULL    REFERENCES signald_accounts(uuid) ON DELETE CASCADE,
+		id            TEXT       NOT NULL,
+		category      TEXT       NOT NULL,
+		name          TEXT       NOT NULL,
+		description   TEXT       NOT NULL,
+		sprite6       TEXT       NOT NULL,
+
+		PRIMARY KEY (account_uuid, id)
 	);
 `
 )

cmd/signaldctl/cmd/device/root.go

@@ -25,7 +25,7 @@ import (
 
 var DeviceCmd = &cobra.Command{
 	Use:     "device",
-	Aliases: []string{"msg", "devices"},
+	Aliases: []string{"devices"},
 }
 
 func init() {

cmd/signaldctl/cmd/group/preview/preview.go

@@ -0,0 +1,102 @@
+// Copyright © 2021 Finn Herzfeld <finn@janky.solutions>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package preview
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/jedib0t/go-pretty/v6/table"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+
+	"gitlab.com/signald/signald-go/cmd/signaldctl/common"
+	"gitlab.com/signald/signald-go/cmd/signaldctl/config"
+	"gitlab.com/signald/signald-go/signald/client-protocol/v1"
+)
+
+var (
+	account         string
+	joinURL         string
+	PreviewGroupCmd = &cobra.Command{
+		Use:   "preview https://signal.group/...",
+		Short: "preview information about a group without joining",
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if account == "" {
+				account = config.Config.DefaultAccount
+			}
+			if account == "" {
+				common.Must(cmd.Help())
+				log.Fatal("No account specified. Please specify with --account or set a default")
+			}
+
+			if len(args) == 0 {
+				return errors.New("please specify signald.group URL")
+			}
+			joinURL = args[0]
+
+			if !strings.HasPrefix(joinURL, "https://signal.group/#") {
+				return errors.New("invalid group link (must start with https://signal.group/#")
+			}
+
+			return nil
+		},
+		RunE: func(_ *cobra.Command, args []string) error {
+			go common.Signald.Listen(nil)
+			req := v1.GroupLinkInfoRequest{
+				Account: account,
+				Uri:     joinURL,
+			}
+
+			info, err := req.Submit(common.Signald)
+			if err != nil {
+				return err
+			}
+
+			switch common.OutputFormat {
+			case common.OutputFormatJSON:
+				return json.NewEncoder(os.Stdout).Encode(info)
+			case common.OutputFormatYAML:
+				return yaml.NewEncoder(os.Stdout).Encode(info)
+			case common.OutputFormatDefault, common.OutputFormatTable:
+				t := table.NewWriter()
+				t.SetOutputMirror(os.Stdout)
+
+				joinApproval := "unknown"
+				switch info.AddFromInviteLink {
+				case 3:
+					joinApproval = "yes"
+				case 1:
+					joinApproval = "no"
+				}
+
+				t.AppendRows([]table.Row{
+					table.Row{"Title", info.Title},
+					table.Row{"Group ID", info.GroupID},
+					table.Row{"Member Count", info.MemberCount},
+					table.Row{"Membership Approval", joinApproval},
+				})
+				common.StylizeTable(t)
+				t.Render()
+			}
+
+			return nil
+		},
+	}
+)

cmd/signaldctl/cmd/group/root.go

@@ -24,6 +24,7 @@ import (
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/group/join"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/group/leave"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/group/list"
+	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/group/preview"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/group/removemember"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/group/show"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/cmd/group/update"
@@ -41,6 +42,7 @@ func init() {
 	GroupCmd.AddCommand(join.JoinGroupCmd)
 	GroupCmd.AddCommand(leave.LeaveGroupCmd)
 	GroupCmd.AddCommand(list.ListGroupCmd)
+	GroupCmd.AddCommand(preview.PreviewGroupCmd)
 	GroupCmd.AddCommand(removemember.RemoveMemberCmd)
 	GroupCmd.AddCommand(show.ShowGroupCmd)
 	GroupCmd.AddCommand(update.UpdateGroupCmd)

cmd/signaldctl/cmd/message/send/send-message.go

@@ -18,8 +18,11 @@ package send
 import (
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -29,7 +32,11 @@ import (
 
 	"gitlab.com/signald/signald-go/cmd/signaldctl/common"
 	"gitlab.com/signald/signald-go/cmd/signaldctl/config"
-	"gitlab.com/signald/signald-go/signald/client-protocol/v1"
+	v1 "gitlab.com/signald/signald-go/signald/client-protocol/v1"
+)
+
+const (
+	CAPTCHA_HELPER = "signal-captcha-helper"
 )
 
 var (
@@ -37,9 +44,11 @@ var (
 	toAddress     *v1.JsonAddress
 	toGroup       string
 	attachments   []string
+	message       string
+	captchaHelper bool
 
 	SendMessageCmd = &cobra.Command{
-		Use:   "send <group id | phone number> <message>",
+		Use:   "send {group id | phone number} [message]",
 		Short: "send a message",
 		PreRun: func(cmd *cobra.Command, args []string) {
 			if account == "" {
@@ -49,9 +58,20 @@ var (
 				common.Must(cmd.Help())
 				log.Fatal("No account specified. Please specify with --account or set a default")
 			}
-			if len(args) < 2 {
+			if len(args) < 1 {
 				common.Must(cmd.Help())
-				log.Fatal("must specify both destination (either group id or phone number) and message")
+				log.Fatal("must specify both destination (either group id or phone number)")
+			}
+
+			if len(args) == 1 {
+				messageBytes, err := ioutil.ReadAll(os.Stdin)
+				if err != nil {
+					log.Println("error reading message from stdin, perhaps you meant to include it in the command line arguments?")
+					panic(err)
+				}
+				message = string(messageBytes)
+			} else {
+				message = strings.Join(args[1:], " ")
 			}
 			to, err := common.StringToAddress(args[0])
 			if err != nil {
@@ -65,7 +85,7 @@ var (
 
 			req := v1.SendRequest{
 				Username:         account,
-				MessageBody:      strings.Join(args[1:], " "),
+				MessageBody:      message,
 				Attachments:      []*v1.JsonAttachment{},
 				RecipientAddress: toAddress,
 				RecipientGroupID: toGroup,
@@ -84,6 +104,48 @@ var (
 			if err != nil {
 				log.Fatal("error sending request to signald: ", err)
 			}
+
+			resends := []*v1.JsonAddress{}
+
+			for _, result := range resp.Results {
+				if result.ProofRequiredFailure != nil {
+					if captchaHelper {
+						err = runCaptchaHelper(result.ProofRequiredFailure.Token)
+						if err != nil {
+							log.Println("error running captcha helper: ", err)
+						}
+						resends = append(resends, result.Address)
+					}
+				}
+			}
+
+			if len(resends) > 0 {
+				resendReq := v1.SendRequest{
+					Username:         req.Username,
+					MessageBody:      req.MessageBody,
+					Attachments:      req.Attachments,
+					RecipientGroupID: req.RecipientGroupID,
+					Members:          resends,
+					Timestamp:        resp.Timestamp,
+				}
+				resendResponse, err := resendReq.Submit(common.Signald)
+				if err != nil {
+					log.Println("error resending messages: ", err)
+				} else {
+					for i, originalResult := range resp.Results {
+						if originalResult.ProofRequiredFailure == nil {
+							continue
+						}
+
+						for _, result := range resendResponse.Results {
+							if result.Address.UUID == originalResult.Address.UUID {
+								resp.Results[i] = result
+							}
+						}
+					}
+				}
+			}
+
 			switch common.OutputFormat {
 			case common.OutputFormatJSON:
 				err := json.NewEncoder(os.Stdout).Encode(resp)
@@ -135,7 +197,55 @@ var (
 	}
 )
 
+func runCaptchaHelper(challenge string) error {
+	if !captchaHelper {
+		return nil
+	}
+
+	_, err := exec.LookPath(CAPTCHA_HELPER)
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command(CAPTCHA_HELPER, "--challenge")
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return err
+	}
+
+	captchaToken := new(strings.Builder)
+	_, err = io.Copy(captchaToken, stdout)
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return err
+	}
+
+	req := v1.SubmitChallengeRequest{
+		Account:      account,
+		CaptchaToken: captchaToken.String(),
+		Challenge:    challenge,
+	}
+
+	err = req.Submit(common.Signald)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func init() {
 	SendMessageCmd.Flags().StringVarP(&account, "account", "a", "", "local account to use")
+	SendMessageCmd.Flags().BoolVarP(&captchaHelper, "captcha-helper", "c", false, "Invoke signal-captcha-helper and process the response when a push challenge response appears. After completing the challenge, the message will be redelivered to the failed recipient")
 	SendMessageCmd.Flags().StringSliceVarP(&attachments, "attachment", "A", []string{}, "attach a file to your outbound message. may be specified multiple times.")
 }

cmd/signaldctl/cmd/session/root.go

@@ -15,7 +15,6 @@
 
 package session
 
-
 import (
 	"github.com/spf13/cobra"
 )

cmd/signaldctl/cmd/subscribe.go

@@ -0,0 +1,96 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"gitlab.com/signald/signald-go/cmd/signaldctl/common"
+	"gitlab.com/signald/signald-go/cmd/signaldctl/config"
+	client_protocol "gitlab.com/signald/signald-go/signald/client-protocol"
+	v1 "gitlab.com/signald/signald-go/signald/client-protocol/v1"
+)
+
+var (
+	accountIdentifier string
+	subscribeCmd      = &cobra.Command{
+		Use:   "subscribe",
+		Short: "subscribe to incoming messages from signald",
+		Long: `subscribe to incoming messages from signald.
+
+		if no default account is set, the -a/--account argument is required.
+
+		the default output format (-o default) is being worked on, subject to change and should not be relied upon. Thoughts? come by the #signald IRC/matrix room
+
+		if you want to future-proof your scripts, use json output (-o json) and parse it.`,
+		PreRun: func(cmd *cobra.Command, args []string) {
+			if accountIdentifier == "" {
+				accountIdentifier = config.Config.DefaultAccount
+			}
+			if accountIdentifier == "" {
+				common.Must(cmd.Help())
+				log.Fatal("No account specified. Please specify with --account or set a default")
+			}
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			incoming := make(chan client_protocol.BasicResponse)
+			go common.Signald.Listen(incoming)
+
+			req := v1.SubscribeRequest{Account: accountIdentifier}
+			err := req.Submit(common.Signald)
+			if err != nil {
+				panic(err)
+			}
+
+			for msg := range incoming {
+				switch common.OutputFormat {
+				case common.OutputFormatDefault:
+					if msg.Type != "IncomingMessage" {
+						continue
+					}
+					var data v1.IncomingMessage
+					err := json.Unmarshal(msg.Data, &data)
+					if err != nil {
+						continue
+					}
+
+					if data.DataMessage == nil {
+						continue
+					}
+
+					group := "-"
+					if data.DataMessage.GroupV2 != nil {
+						group = fmt.Sprintf("%s|%d", data.DataMessage.GroupV2.ID, data.DataMessage.GroupV2.Revision)
+					}
+
+					attachment := "-"
+					if len(data.DataMessage.Attachments) > 0 {
+						attachment = data.DataMessage.Attachments[0].StoredFilename
+					}
+
+					body := "-"
+					if data.DataMessage.Body != "" {
+						body = data.DataMessage.Body
+					}
+					fmt.Println(data.DataMessage.Timestamp, data.Account, data.Source.UUID, group, attachment, body)
+				case common.OutputFormatJSON:
+					err := json.NewEncoder(os.Stdout).Encode(msg)
+					if err != nil {
+						panic(err)
+					}
+				default:
+					log.Fatal("unsupported output format")
+				}
+			}
+			return nil
+		},
+	}
+)
+
+func init() {
+	subscribeCmd.Flags().StringVarP(&accountIdentifier, "account", "a", "", "the signald account to use")
+	RootCmd.AddCommand(subscribeCmd)
+}

cmd/signaldctl/config/config.go

@@ -18,6 +18,7 @@ package config
 import (
 	"log"
 	"os"
+	"path"
 
 	"gopkg.in/yaml.v2"
 )
@@ -54,6 +55,9 @@ func Load() error {
 }
 
 func Save() error {
+	if err := os.MkdirAll(path.Dir(Path), 0770); err != nil {
+		return err
+	}
 	f, err := os.Create(Path)
 	if err != nil {
 		return err

signald/client-protocol/protocol.go

@@ -5,10 +5,10 @@ import (
 )
 
 type BasicResponse struct {
-	ID        string
+	ID        string          `json:"id,omitempty"`
-	Type      string
+	Type      string          `json:"type,omitempty"`
-	ErrorType string
+	ErrorType string          `json:"error_type,omitempty"`
-	Error     json.RawMessage
+	Error     json.RawMessage `json:"error,omitempty"`
-	Data      json.RawMessage
+	Data      json.RawMessage `json:"data,omitempty"`
-	Account   string
+	Account   string          `json:"account,omitempty"`
 }

signald/client-protocol/v0/structs.go

@@ -291,6 +291,7 @@ type OfferMessage struct {
 }
 
 type Optional struct {
+	Empty   bool `json:"empty,omitempty" yaml:"empty,omitempty"`
 	Present bool `json:"present,omitempty" yaml:"present,omitempty"`
 }
 

signald/client-protocol/v1/errors.go

@@ -81,6 +81,13 @@ func mkerr(response client_protocol.BasicResponse) error {
 			return err
 		}
 		return result
+	case "GroupPatchNotAcceptedError":
+		result := GroupPatchNotAcceptedError{}
+		err := json.Unmarshal(response.Error, &result)
+		if err != nil {
+			return err
+		}
+		return result
 	case "GroupVerificationError":
 		result := GroupVerificationError{}
 		err := json.Unmarshal(response.Error, &result)
@@ -207,6 +214,13 @@ func mkerr(response client_protocol.BasicResponse) error {
 			return err
 		}
 		return result
+	case "ProtocolInvalidKeyIdError":
+		result := ProtocolInvalidKeyIdError{}
+		err := json.Unmarshal(response.Error, &result)
+		if err != nil {
+			return err
+		}
+		return result
 	case "ProtocolInvalidMessageError":
 		result := ProtocolInvalidMessageError{}
 		err := json.Unmarshal(response.Error, &result)
@@ -214,6 +228,13 @@ func mkerr(response client_protocol.BasicResponse) error {
 			return err
 		}
 		return result
+	case "ProtocolNoSessionError":
+		result := ProtocolNoSessionError{}
+		err := json.Unmarshal(response.Error, &result)
+		if err != nil {
+			return err
+		}
+		return result
 	case "RateLimitError":
 		result := RateLimitError{}
 		err := json.Unmarshal(response.Error, &result)
@@ -221,6 +242,13 @@ func mkerr(response client_protocol.BasicResponse) error {
 			return err
 		}
 		return result
+	case "SQLError":
+		result := SQLError{}
+		err := json.Unmarshal(response.Error, &result)
+		if err != nil {
+			return err
+		}
+		return result
 	case "ScanTimeoutError":
 		result := ScanTimeoutError{}
 		err := json.Unmarshal(response.Error, &result)
@@ -256,6 +284,13 @@ func mkerr(response client_protocol.BasicResponse) error {
 			return err
 		}
 		return result
+	case "UnsupportedGroupError":
+		result := UnsupportedGroupError{}
+		err := json.Unmarshal(response.Error, &result)
+		if err != nil {
+			return err
+		}
+		return result
 	case "UntrustedIdentityError":
 		result := UntrustedIdentityError{}
 		err := json.Unmarshal(response.Error, &result)
@@ -309,7 +344,7 @@ func (e AttachmentTooLargeError) Error() string {
 	return e.Message
 }
 
-// AuthorizationFailedError: indicates the server rejected our credentials. Typically means the linked device was removed by the primary device, or that the account was re-registered
+// AuthorizationFailedError: Indicates the server rejected our credentials or a failed group update. Typically means the linked device was removed by the primary device, or that the account was re-registered. For group updates, this can indicate that we lack permissions.
 type AuthorizationFailedError struct {
 	Message string `json:"message,omitempty" yaml:"message,omitempty"`
 }
@@ -360,6 +395,15 @@ func (e GroupNotActiveError) Error() string {
 	return e.Message
 }
 
+// GroupPatchNotAcceptedError: Indicates the server rejected our group update. This can be due to errors such as trying to add a user that's already in the group.
+type GroupPatchNotAcceptedError struct {
+	Message string `json:"message,omitempty" yaml:"message,omitempty"`
+}
+
+func (e GroupPatchNotAcceptedError) Error() string {
+	return e.Message
+}
+
 type GroupVerificationError struct {
 	Message string `json:"message,omitempty" yaml:"message,omitempty"`
 }
@@ -511,6 +555,19 @@ func (e ProofRequiredError) Error() string {
 	return e.Message
 }
 
+type ProtocolInvalidKeyIdError struct {
+	ContentHint  int32  `json:"content_hint,omitempty" yaml:"content_hint,omitempty"`
+	GroupId      string `json:"group_id,omitempty" yaml:"group_id,omitempty"`
+	Message      string `json:"message,omitempty" yaml:"message,omitempty"`
+	Sender       string `json:"sender,omitempty" yaml:"sender,omitempty"`
+	SenderDevice int32  `json:"sender_device,omitempty" yaml:"sender_device,omitempty"`
+	Timestamp    int64  `json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
+}
+
+func (e ProtocolInvalidKeyIdError) Error() string {
+	return e.Message
+}
+
 type ProtocolInvalidMessageError struct {
 	ContentHint  int32  `json:"content_hint,omitempty" yaml:"content_hint,omitempty"`
 	GroupId      string `json:"group_id,omitempty" yaml:"group_id,omitempty"`
@@ -524,6 +581,19 @@ func (e ProtocolInvalidMessageError) Error() string {
 	return e.Message
 }
 
+type ProtocolNoSessionError struct {
+	ContentHint  int32  `json:"content_hint,omitempty" yaml:"content_hint,omitempty"`
+	GroupId      string `json:"group_id,omitempty" yaml:"group_id,omitempty"`
+	Message      string `json:"message,omitempty" yaml:"message,omitempty"`
+	Sender       string `json:"sender,omitempty" yaml:"sender,omitempty"`
+	SenderDevice int32  `json:"sender_device,omitempty" yaml:"sender_device,omitempty"`
+	Timestamp    int64  `json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
+}
+
+func (e ProtocolNoSessionError) Error() string {
+	return e.Message
+}
+
 type RateLimitError struct {
 	Message string `json:"message,omitempty" yaml:"message,omitempty"`
 }
@@ -532,6 +602,14 @@ func (e RateLimitError) Error() string {
 	return e.Message
 }
 
+type SQLError struct {
+	Message string `json:"message,omitempty" yaml:"message,omitempty"`
+}
+
+func (e SQLError) Error() string {
+	return e.Message
+}
+
 type ScanTimeoutError struct {
 	Message string `json:"message,omitempty" yaml:"message,omitempty"`
 }
@@ -574,6 +652,15 @@ func (e UnregisteredUserError) Error() string {
 	return e.Message
 }
 
+// UnsupportedGroupError: returned in response to use v1 groups, which are no longer supported
+type UnsupportedGroupError struct {
+	Message string `json:"message,omitempty" yaml:"message,omitempty"`
+}
+
+func (e UnsupportedGroupError) Error() string {
+	return e.Message
+}
+
 type UntrustedIdentityError struct {
 	Identifier  string        `json:"identifier,omitempty" yaml:"identifier,omitempty"`
 	IdentityKey **IdentityKey `json:"identity_key,omitempty" yaml:"identity_key,omitempty"`

signald/client-protocol/v1/requests.go

@@ -16,15 +16,14 @@ func (r *AcceptInvitationRequest) Submit(conn *signald.Signald) (response JsonGr
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -49,15 +48,14 @@ func (r *AddLinkedDeviceRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -75,15 +73,14 @@ func (r *AddServerRequest) Submit(conn *signald.Signald) (response string, err e
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -108,14 +105,45 @@ func (r *ApproveMembershipRequest) Submit(conn *signald.Signald) (response JsonG
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
+	rawResponse := <-responseChannel
+	if rawResponse.Error != nil {
+		err = mkerr(rawResponse)
+		return
+	}
+
+	err = json.Unmarshal(rawResponse.Data, &response)
+	if err != nil {
+		rawResponseJson, _ := rawResponse.Data.MarshalJSON()
+		log.Println("signald-go: error unmarshalling response from signald of type", rawResponse.Type, string(rawResponseJson))
+		return
+	}
+
+	return response, nil
+
+}
+
+// Submit: Bans users from a group. This works even if the users aren't in the group. If they are currently in the group, they will also be removed.
+func (r *BanUserRequest) Submit(conn *signald.Signald) (response JsonGroupV2Info, err error) {
+	r.Version = "v1"
+	r.Type = "ban_user"
+	if r.ID == "" {
+		r.ID = signald.GenerateID()
+	}
 	responseChannel := conn.GetResponseListener(r.ID)
 	defer conn.CloseResponseListener(r.ID)
+	err = conn.RawRequest(r)
+	if err != nil {
+		log.Println("signald-go: error submitting request to signald")
+		return
+	}
 
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
@@ -140,15 +168,14 @@ func (r *CreateGroupRequest) Submit(conn *signald.Signald) (response JsonGroupV2
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -173,15 +200,14 @@ func (r *DeleteAccountRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -198,15 +224,14 @@ func (r *RemoveServerRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -224,15 +249,14 @@ func (r *FinishLinkRequest) Submit(conn *signald.Signald) (response Account, err
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -257,15 +281,14 @@ func (r *GenerateLinkingURIRequest) Submit(conn *signald.Signald) (response Link
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -290,15 +313,14 @@ func (r *GetAllIdentities) Submit(conn *signald.Signald) (response AllIdentityKe
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -323,14 +345,45 @@ func (r *GetGroupRequest) Submit(conn *signald.Signald) (response JsonGroupV2Inf
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
+	rawResponse := <-responseChannel
+	if rawResponse.Error != nil {
+		err = mkerr(rawResponse)
+		return
+	}
+
+	err = json.Unmarshal(rawResponse.Data, &response)
+	if err != nil {
+		rawResponseJson, _ := rawResponse.Data.MarshalJSON()
+		log.Println("signald-go: error unmarshalling response from signald of type", rawResponse.Type, string(rawResponseJson))
+		return
+	}
+
+	return response, nil
+
+}
+
+// Submit: Query the server for group revision history. The history contains information about the changes between each revision and the user that made the change.
+func (r *GetGroupRevisionPagesRequest) Submit(conn *signald.Signald) (response GroupHistoryPage, err error) {
+	r.Version = "v1"
+	r.Type = "get_group_revision_pages"
+	if r.ID == "" {
+		r.ID = signald.GenerateID()
+	}
 	responseChannel := conn.GetResponseListener(r.ID)
 	defer conn.CloseResponseListener(r.ID)
+	err = conn.RawRequest(r)
+	if err != nil {
+		log.Println("signald-go: error submitting request to signald")
+		return
+	}
 
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
@@ -356,15 +409,14 @@ func (r *GetIdentitiesRequest) Submit(conn *signald.Signald) (response IdentityK
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -389,15 +441,14 @@ func (r *GetLinkedDevicesRequest) Submit(conn *signald.Signald) (response Linked
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -422,15 +473,14 @@ func (r *GetProfileRequest) Submit(conn *signald.Signald) (response Profile, err
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -455,15 +505,14 @@ func (r *RemoteConfigRequest) Submit(conn *signald.Signald) (response RemoteConf
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -487,15 +536,14 @@ func (r *GetServersRequest) Submit(conn *signald.Signald) (response ServerList,
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -520,15 +568,14 @@ func (r *GroupLinkInfoRequest) Submit(conn *signald.Signald) (response JsonGroup
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -553,15 +600,14 @@ func (r *IsIdentifierRegisteredRequest) Submit(conn *signald.Signald) (response
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -586,15 +632,14 @@ func (r *JoinGroupRequest) Submit(conn *signald.Signald) (response JsonGroupJoin
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -618,15 +663,14 @@ func (r *LeaveGroupRequest) Submit(conn *signald.Signald) (response GroupInfo, e
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -651,15 +695,14 @@ func (r *ListAccountsRequest) Submit(conn *signald.Signald) (response AccountLis
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -683,15 +726,14 @@ func (r *ListContactsRequest) Submit(conn *signald.Signald) (response ProfileLis
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -715,15 +757,14 @@ func (r *ListGroupsRequest) Submit(conn *signald.Signald) (response GroupList, e
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -747,15 +788,14 @@ func (r *MarkReadRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -773,15 +813,14 @@ func (r *ReactRequest) Submit(conn *signald.Signald) (response SendResponse, err
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -806,15 +845,14 @@ func (r *RefuseMembershipRequest) Submit(conn *signald.Signald) (response JsonGr
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -839,15 +877,14 @@ func (r *RegisterRequest) Submit(conn *signald.Signald) (response Account, err e
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -872,15 +909,14 @@ func (r *RemoteDeleteRequest) Submit(conn *signald.Signald) (response SendRespon
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -905,15 +941,14 @@ func (r *RemoveLinkedDeviceRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -931,15 +966,14 @@ func (r *RequestSyncRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -957,15 +991,14 @@ func (r *ResetSessionRequest) Submit(conn *signald.Signald) (response SendRespon
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -990,15 +1023,14 @@ func (r *ResolveAddressRequest) Submit(conn *signald.Signald) (response JsonAddr
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1022,15 +1054,14 @@ func (r *SendRequest) Submit(conn *signald.Signald) (response SendResponse, err
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1055,14 +1086,45 @@ func (r *SendPaymentRequest) Submit(conn *signald.Signald) (response SendRespons
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
+	rawResponse := <-responseChannel
+	if rawResponse.Error != nil {
+		err = mkerr(rawResponse)
+		return
+	}
+
+	err = json.Unmarshal(rawResponse.Data, &response)
+	if err != nil {
+		rawResponseJson, _ := rawResponse.Data.MarshalJSON()
+		log.Println("signald-go: error unmarshalling response from signald of type", rawResponse.Type, string(rawResponseJson))
+		return
+	}
+
+	return response, nil
+
+}
+
+// Submit: Sends a sync message to the account's devices
+func (r *SendSyncMessageRequest) Submit(conn *signald.Signald) (response JsonSendMessageResult, err error) {
+	r.Version = "v1"
+	r.Type = "send_sync_message"
+	if r.ID == "" {
+		r.ID = signald.GenerateID()
+	}
 	responseChannel := conn.GetResponseListener(r.ID)
 	defer conn.CloseResponseListener(r.ID)
+	err = conn.RawRequest(r)
+	if err != nil {
+		log.Println("signald-go: error submitting request to signald")
+		return
+	}
 
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
@@ -1088,15 +1150,14 @@ func (r *SetDeviceNameRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1114,15 +1175,14 @@ func (r *SetExpirationRequest) Submit(conn *signald.Signald) (response SendRespo
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1146,15 +1206,14 @@ func (r *SetProfile) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1171,15 +1230,14 @@ func (r *SubmitChallengeRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1197,15 +1255,14 @@ func (r *SubscribeRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1223,15 +1280,14 @@ func (r *TrustRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1249,15 +1305,14 @@ func (r *TypingRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1268,6 +1323,38 @@ func (r *TypingRequest) Submit(conn *signald.Signald) (err error) {
 
 }
 
+// Submit: Unbans users from a group.
+func (r *UnbanUserRequest) Submit(conn *signald.Signald) (response JsonGroupV2Info, err error) {
+	r.Version = "v1"
+	r.Type = "unban_user"
+	if r.ID == "" {
+		r.ID = signald.GenerateID()
+	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
+	err = conn.RawRequest(r)
+	if err != nil {
+		log.Println("signald-go: error submitting request to signald")
+		return
+	}
+
+	rawResponse := <-responseChannel
+	if rawResponse.Error != nil {
+		err = mkerr(rawResponse)
+		return
+	}
+
+	err = json.Unmarshal(rawResponse.Data, &response)
+	if err != nil {
+		rawResponseJson, _ := rawResponse.Data.MarshalJSON()
+		log.Println("signald-go: error unmarshalling response from signald of type", rawResponse.Type, string(rawResponseJson))
+		return
+	}
+
+	return response, nil
+
+}
+
 // Submit: See subscribe for more info
 func (r *UnsubscribeRequest) Submit(conn *signald.Signald) (err error) {
 	r.Version = "v1"
@@ -1275,15 +1362,14 @@ func (r *UnsubscribeRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1301,15 +1387,14 @@ func (r *UpdateContactRequest) Submit(conn *signald.Signald) (response Profile,
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1334,15 +1419,14 @@ func (r *UpdateGroupRequest) Submit(conn *signald.Signald) (response GroupInfo,
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1367,15 +1451,14 @@ func (r *VerifyRequest) Submit(conn *signald.Signald) (response Account, err err
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1399,15 +1482,14 @@ func (r *VersionRequest) Submit(conn *signald.Signald) (response JsonVersionMess
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)
@@ -1432,15 +1514,14 @@ func (r *WaitForScanRequest) Submit(conn *signald.Signald) (err error) {
 	if r.ID == "" {
 		r.ID = signald.GenerateID()
 	}
+	responseChannel := conn.GetResponseListener(r.ID)
+	defer conn.CloseResponseListener(r.ID)
 	err = conn.RawRequest(r)
 	if err != nil {
 		log.Println("signald-go: error submitting request to signald")
 		return
 	}
 
-	responseChannel := conn.GetResponseListener(r.ID)
-	defer conn.CloseResponseListener(r.ID)
-
 	rawResponse := <-responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)

signald/client-protocol/v1/structs.go

@@ -25,6 +25,7 @@ type Account struct {
 	Address   *JsonAddress `json:"address,omitempty" yaml:"address,omitempty"`       // The address of this account
 	DeviceId  int32        `json:"device_id,omitempty" yaml:"device_id,omitempty"`   // The Signal device ID. Official Signal mobile clients (iPhone and Android) have device ID = 1, while linked devices such as Signal Desktop or Signal iPad have higher device IDs.
 	Pending   bool         `json:"pending,omitempty" yaml:"pending,omitempty"`       // indicates the account has not completed registration
+	Pni       string       `json:"pni,omitempty" yaml:"pni,omitempty"`
 }
 
 type AccountList struct {
@@ -62,6 +63,19 @@ type ApproveMembershipRequest struct {
 	Members []*JsonAddress `json:"members,omitempty" yaml:"members,omitempty"` // list of requesting members to approve
 }
 
+// BanUserRequest: Bans users from a group. This works even if the users aren't in the group. If they are currently in the group, they will also be removed.
+type BanUserRequest struct {
+	Request
+	Account string         `json:"account,omitempty" yaml:"account,omitempty"` // The account to interact with
+	GroupId string         `json:"group_id,omitempty" yaml:"group_id,omitempty"`
+	Users   []*JsonAddress `json:"users,omitempty" yaml:"users,omitempty"` // List of users to ban
+}
+
+type BannedGroupMember struct {
+	Timestamp int64  `json:"timestamp,omitempty" yaml:"timestamp,omitempty"` // Timestamp as milliseconds since Unix epoch of when the user was banned. This field is set by the server.
+	UUID      string `json:"uuid,omitempty" yaml:"uuid,omitempty"`
+}
+
 // BooleanMessage: A message containing a single boolean, usually as a response
 type BooleanMessage struct {
 	Value bool `json:"value,omitempty" yaml:"value,omitempty"`
@@ -85,9 +99,10 @@ type Capabilities struct {
 	AnnouncementGroup bool `json:"announcement_group,omitempty" yaml:"announcement_group,omitempty"`
 	ChangeNumber      bool `json:"change_number,omitempty" yaml:"change_number,omitempty"`
 	Gv1Migration      bool `json:"gv1-migration,omitempty" yaml:"gv1-migration,omitempty"`
-	Gv2               bool `json:"gv2,omitempty" yaml:"gv2,omitempty"`
+	Gv2               bool `json:"gv2,omitempty" yaml:"gv2,omitempty"` // this capability is deprecated and will always be true
 	SenderKey         bool `json:"sender_key,omitempty" yaml:"sender_key,omitempty"`
 	Storage           bool `json:"storage,omitempty" yaml:"storage,omitempty"`
+	Stories           bool `json:"stories,omitempty" yaml:"stories,omitempty"`
 }
 
 // ClientMessageWrapper: Wraps all incoming messages sent to the client after a v1 subscribe request is issued
@@ -127,6 +142,7 @@ type DeviceInfo struct {
 type FinishLinkRequest struct {
 	Request
 	DeviceName string `json:"device_name,omitempty" yaml:"device_name,omitempty"`
+	Overwrite  bool   `json:"overwrite,omitempty" yaml:"overwrite,omitempty"` // overwrite existing account data if the phone number conflicts. false by default
 	SessionId  string `json:"session_id,omitempty" yaml:"session_id,omitempty"`
 }
 
@@ -150,6 +166,15 @@ type GetGroupRequest struct {
 	Revision int32  `json:"revision,omitempty" yaml:"revision,omitempty"` // the latest known revision, default value (-1) forces fetch from server
 }
 
+// GetGroupRevisionPagesRequest: Query the server for group revision history. The history contains information about the changes between each revision and the user that made the change.
+type GetGroupRevisionPagesRequest struct {
+	Request
+	Account              string `json:"account,omitempty" yaml:"account,omitempty"`             // The account to interact with
+	FromRevision         int32  `json:"from_revision,omitempty" yaml:"from_revision,omitempty"` // The revision to start the pages from. Note that if this is lower than the revision you joined the group, an AuthorizationFailedError is returned.
+	GroupId              string `json:"group_id,omitempty" yaml:"group_id,omitempty"`
+	IncludeFirstRevision bool   `json:"include_first_revision,omitempty" yaml:"include_first_revision,omitempty"` // Whether to include the first state in the returned pages (default false)
+}
+
 // GetIdentitiesRequest: Get information about a known keys for a particular address
 type GetIdentitiesRequest struct {
 	Request
@@ -175,6 +200,11 @@ type GetServersRequest struct {
 	Request
 }
 
+type Gradient struct {
+	EndColor   string `json:"end_color,omitempty" yaml:"end_color,omitempty"`
+	StartColor string `json:"start_color,omitempty" yaml:"start_color,omitempty"`
+}
+
 // GroupAccessControl: group access control settings. Options for each controlled action are: UNKNOWN, ANY, MEMBER, ADMINISTRATOR, UNSATISFIABLE and UNRECOGNIZED
 type GroupAccessControl struct {
 	Attributes string `json:"attributes,omitempty" yaml:"attributes,omitempty"` // who can edit group info
@@ -182,6 +212,42 @@ type GroupAccessControl struct {
 	Members    string `json:"members,omitempty" yaml:"members,omitempty"`       // who can add members
 }
 
+// GroupChange: Represents a group change made by a user. This can also represent request link invites. Only the fields relevant to the group change performed will be set. Note that in signald, group changes are currently only received from incoming messages from a message subscription.
+type GroupChange struct {
+	DeleteMembers            []*JsonAddress           `json:"delete_members,omitempty" yaml:"delete_members,omitempty"` // Represents users that have been removed from the group. This can be from admins removing users, or users choosing to leave the group
+	DeletePendingMembers     []*JsonAddress           `json:"delete_pending_members,omitempty" yaml:"delete_pending_members,omitempty"`
+	DeleteRequestingMembers  []*JsonAddress           `json:"delete_requesting_members,omitempty" yaml:"delete_requesting_members,omitempty"`
+	Editor                   *JsonAddress             `json:"editor,omitempty" yaml:"editor,omitempty"`                               // The user that made the change.
+	ModifiedProfileKeys      []*GroupMember           `json:"modified_profile_keys,omitempty" yaml:"modified_profile_keys,omitempty"` // Represents users that have rotated their profile key. Note that signald currently does not expose profile keys to clients. The joined revision property will always be 0 in this list.
+	ModifyMemberRoles        []*GroupMember           `json:"modify_member_roles,omitempty" yaml:"modify_member_roles,omitempty"`     // Represents users with their new, modified role.
+	NewAccessControl         *GroupAccessControl      `json:"new_access_control,omitempty" yaml:"new_access_control,omitempty"`       // If not null, then this group change modified one of the access controls. Some of the properties in here will be null.
+	NewAvatar                bool                     `json:"new_avatar,omitempty" yaml:"new_avatar,omitempty"`                       // Whether this group change changed the avatar.
+	NewBannedMembers         []*BannedGroupMember     `json:"new_banned_members,omitempty" yaml:"new_banned_members,omitempty"`
+	NewDescription           string                   `json:"new_description,omitempty" yaml:"new_description,omitempty"`
+	NewInviteLinkPassword    bool                     `json:"new_invite_link_password,omitempty" yaml:"new_invite_link_password,omitempty"`   // Whether this group change involved resetting the group invite link.
+	NewIsAnnouncementGroup   string                   `json:"new_is_announcement_group,omitempty" yaml:"new_is_announcement_group,omitempty"` // Whether this change affected the announcement group setting. Possible values are UNKNOWN, ENABLED or DISABLED
+	NewMembers               []*GroupMember           `json:"new_members,omitempty" yaml:"new_members,omitempty"`                             // Represents users have been added to the group. This can be from group members adding users, or a users joining via a group link that required no approval.
+	NewPendingMembers        []*GroupPendingMember    `json:"new_pending_members,omitempty" yaml:"new_pending_members,omitempty"`             // Represents a user that has been invited to the group by another user.
+	NewRequestingMembers     []*GroupRequestingMember `json:"new_requesting_members,omitempty" yaml:"new_requesting_members,omitempty"`       // Represents users that have requested to join the group via the group link. Note that members requesting to join might not necessarily have the list of users in the group, so they won't be able to send a peer-to-peer group update message to inform users of their request to join. Other users in the group may inform us that the revision has increased, but the members requesting access will have to be obtained from the server instead (which signald will handle). For now, a get_group request has to be made to get the users that have requested to join the group.
+	NewTimer                 int32                    `json:"new_timer,omitempty" yaml:"new_timer,omitempty"`                                 // New disappearing messages timer value.
+	NewTitle                 string                   `json:"new_title,omitempty" yaml:"new_title,omitempty"`
+	NewUnbannedMembers       []*BannedGroupMember     `json:"new_unbanned_members,omitempty" yaml:"new_unbanned_members,omitempty"`
+	PromotePendingMembers    []*GroupMember           `json:"promote_pending_members,omitempty" yaml:"promote_pending_members,omitempty"`
+	PromoteRequestingMembers []*GroupMember           `json:"promote_requesting_members,omitempty" yaml:"promote_requesting_members,omitempty"`
+	Revision                 int32                    `json:"revision,omitempty" yaml:"revision,omitempty"` // The group revision that this change brings the group to.
+}
+
+type GroupHistoryEntry struct {
+	Change *GroupChange     `json:"change,omitempty" yaml:"change,omitempty"`
+	Group  *JsonGroupV2Info `json:"group,omitempty" yaml:"group,omitempty"`
+}
+
+// GroupHistoryPage: The result of fetching a group's history along with paging data.
+type GroupHistoryPage struct {
+	PagingData *PagingData          `json:"paging_data,omitempty" yaml:"paging_data,omitempty"`
+	Results    []*GroupHistoryEntry `json:"results,omitempty" yaml:"results,omitempty"`
+}
+
 // GroupInfo: A generic type that is used when the group version is not known
 type GroupInfo struct {
 	V1 *JsonGroupInfo   `json:"v1,omitempty" yaml:"v1,omitempty"`
@@ -197,7 +263,7 @@ type GroupLinkInfoRequest struct {
 
 type GroupList struct {
 	Groups       []*JsonGroupV2Info `json:"groups,omitempty" yaml:"groups,omitempty"`
-	LegacyGroups []*JsonGroupInfo   `json:"legacyGroups,omitempty" yaml:"legacyGroups,omitempty"`
+	LegacyGroups []*JsonGroupInfo   `json:"legacyGroups,omitempty" yaml:"legacyGroups,omitempty"` // list of legacy (v1) groups, no longer supported (will always be empty)
 }
 
 type GroupMember struct {
@@ -206,6 +272,18 @@ type GroupMember struct {
 	UUID           string `json:"uuid,omitempty" yaml:"uuid,omitempty"`
 }
 
+type GroupPendingMember struct {
+	AddedByUuid string `json:"added_by_uuid,omitempty" yaml:"added_by_uuid,omitempty"`
+	Role        string `json:"role,omitempty" yaml:"role,omitempty"` // possible values are: UNKNOWN, DEFAULT, ADMINISTRATOR and UNRECOGNIZED
+	Timestamp   int64  `json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
+	UUID        string `json:"uuid,omitempty" yaml:"uuid,omitempty"`
+}
+
+type GroupRequestingMember struct {
+	Timestamp int64  `json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
+	UUID      string `json:"uuid,omitempty" yaml:"uuid,omitempty"`
+}
+
 type HangupMessage struct {
 	DeviceId int32  `json:"device_id,omitempty" yaml:"device_id,omitempty"`
 	ID       int64  `json:"id,omitempty" yaml:"id,omitempty"`
@@ -244,6 +322,7 @@ type IncomingMessage struct {
 	ServerReceiverTimestamp int64            `json:"server_receiver_timestamp,omitempty" yaml:"server_receiver_timestamp,omitempty"`
 	Source                  *JsonAddress     `json:"source,omitempty" yaml:"source,omitempty"`
 	SourceDevice            int32            `json:"source_device,omitempty" yaml:"source_device,omitempty"`
+	StoryMessage            *StoryMessage    `json:"story_message,omitempty" yaml:"story_message,omitempty"`
 	SyncMessage             *JsonSyncMessage `json:"sync_message,omitempty" yaml:"sync_message,omitempty"`
 	Timestamp               int64            `json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
 	Type                    string           `json:"type,omitempty" yaml:"type,omitempty"`
@@ -302,6 +381,7 @@ type JsonDataMessage struct {
 	Group              *JsonGroupInfo    `json:"group,omitempty" yaml:"group,omitempty"`                               // if the incoming message was sent to a v1 group, information about that group will be here
 	GroupV2            *JsonGroupV2Info  `json:"groupV2,omitempty" yaml:"groupV2,omitempty"`                           // if the incoming message was sent to a v2 group, basic identifying information about that group will be here. If group information changes, JsonGroupV2Info.revision is incremented. If the group revision is higher than previously seen, a client can retrieve the group information by calling get_group.
 	GroupCallUpdate    string            `json:"group_call_update,omitempty" yaml:"group_call_update,omitempty"`       // the eraId string from a group call message update
+	IsExpirationUpdate bool              `json:"is_expiration_update,omitempty" yaml:"is_expiration_update,omitempty"` // whether or not this message changes the expiresInSeconds value for the whole chat. Some messages (remote deletes, reactions, etc) will have expiresInSeconds=0 even though the chat has disappearing messages enabled.
 	Mentions           []*JsonMention    `json:"mentions,omitempty" yaml:"mentions,omitempty"`                         // list of mentions in the message
 	Payment            *Payment          `json:"payment,omitempty" yaml:"payment,omitempty"`                           // details about the MobileCoin payment attached to the message, if present
 	Previews           []*JsonPreview    `json:"previews,omitempty" yaml:"previews,omitempty"`                         // if the incoming message has a link preview, information about that preview will be here
@@ -310,6 +390,7 @@ type JsonDataMessage struct {
 	Reaction           *JsonReaction     `json:"reaction,omitempty" yaml:"reaction,omitempty"`         // if the message adds or removes a reaction to another message, this will indicate what change is being made
 	RemoteDelete       *RemoteDelete     `json:"remoteDelete,omitempty" yaml:"remoteDelete,omitempty"` // if the inbound message is deleting a previously sent message, indicates which message should be deleted
 	Sticker            *v0.JsonSticker   `json:"sticker,omitempty" yaml:"sticker,omitempty"`           // if the incoming message is a sticker, information about the sicker will be here
+	StoryContext       *StoryContext     `json:"story_context,omitempty" yaml:"story_context,omitempty"`
 	Timestamp          int64             `json:"timestamp,omitempty" yaml:"timestamp,omitempty"` // the timestamp that the message was sent at, according to the sender's device. This is used to uniquely identify this message for things like reactions and quotes.
 	ViewOnce           bool              `json:"viewOnce,omitempty" yaml:"viewOnce,omitempty"`   // indicates the message is a view once message. View once messages typically include no body and a single image attachment. Official Signal clients will prevent the user from saving the image, and once the user has viewed the image once they will destroy the image.
 }
@@ -338,7 +419,9 @@ type JsonGroupV2Info struct {
 	AccessControl       *GroupAccessControl  `json:"accessControl,omitempty" yaml:"accessControl,omitempty"` // current access control settings for this group
 	Announcements       string               `json:"announcements,omitempty" yaml:"announcements,omitempty"` // indicates if the group is an announcements group. Only admins are allowed to send messages to announcements groups. Options are UNKNOWN, ENABLED or DISABLED
 	Avatar              string               `json:"avatar,omitempty" yaml:"avatar,omitempty"`               // path to the group's avatar on local disk, if available
+	BannedMembers       []*BannedGroupMember `json:"banned_members,omitempty" yaml:"banned_members,omitempty"`
 	Description         string               `json:"description,omitempty" yaml:"description,omitempty"`
+	GroupChange         *GroupChange         `json:"group_change,omitempty" yaml:"group_change,omitempty"` // Represents a peer-to-peer group change done by a user. Will not be set if the group change signature fails verification. This is usually only set inside of incoming messages.
 	ID                  string               `json:"id,omitempty" yaml:"id,omitempty"`
 	InviteLink          string               `json:"inviteLink,omitempty" yaml:"inviteLink,omitempty"`     // the signal.group link, if applicable
 	MemberDetail        []*GroupMember       `json:"memberDetail,omitempty" yaml:"memberDetail,omitempty"` // detailed member list
@@ -358,31 +441,11 @@ type JsonMention struct {
 	UUID   string `json:"uuid,omitempty" yaml:"uuid,omitempty"`     // The UUID of the account being mentioned
 }
 
-type JsonMessageEnvelope struct {
+// JsonMessageRequestResponseMessage: Responses to message requests from unknown users or groups
-	CallMessage              *v0.JsonCallMessage    `json:"callMessage,omitempty" yaml:"callMessage,omitempty"`
-	DataMessage              *JsonDataMessage       `json:"dataMessage,omitempty" yaml:"dataMessage,omitempty"`
-	HasContent               bool                   `json:"hasContent,omitempty" yaml:"hasContent,omitempty"`
-	HasLegacyMessage         bool                   `json:"hasLegacyMessage,omitempty" yaml:"hasLegacyMessage,omitempty"`
-	IsUnidentifiedSender     bool                   `json:"isUnidentifiedSender,omitempty" yaml:"isUnidentifiedSender,omitempty"`
-	Receipt                  *v0.JsonReceiptMessage `json:"receipt,omitempty" yaml:"receipt,omitempty"`
-	Relay                    string                 `json:"relay,omitempty" yaml:"relay,omitempty"`
-	ServerDeliveredTimestamp int64                  `json:"serverDeliveredTimestamp,omitempty" yaml:"serverDeliveredTimestamp,omitempty"`
-	ServerTimestamp          int64                  `json:"serverTimestamp,omitempty" yaml:"serverTimestamp,omitempty"`
-	Source                   *JsonAddress           `json:"source,omitempty" yaml:"source,omitempty"`
-	SourceDevice             int32                  `json:"sourceDevice,omitempty" yaml:"sourceDevice,omitempty"`
-	SyncMessage              *JsonSyncMessage       `json:"syncMessage,omitempty" yaml:"syncMessage,omitempty"`
-	Timestamp                int64                  `json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
-	TimestampISO             string                 `json:"timestampISO,omitempty" yaml:"timestampISO,omitempty"`
-	Type                     string                 `json:"type,omitempty" yaml:"type,omitempty"`
-	Typing                   *v0.JsonTypingMessage  `json:"typing,omitempty" yaml:"typing,omitempty"`
-	Username                 string                 `json:"username,omitempty" yaml:"username,omitempty"`
-	UUID                     string                 `json:"uuid,omitempty" yaml:"uuid,omitempty"`
-}
-
 type JsonMessageRequestResponseMessage struct {
 	GroupId string       `json:"groupId,omitempty" yaml:"groupId,omitempty"`
 	Person  *JsonAddress `json:"person,omitempty" yaml:"person,omitempty"`
-	Type    string       `json:"type,omitempty" yaml:"type,omitempty"`
+	Type    string       `json:"type,omitempty" yaml:"type,omitempty"` // One of UNKNOWN, ACCEPT, DELETE, BLOCK, BLOCK_AND_DELETE, UNBLOCK_AND_ACCEPT
 }
 
 // JsonPreview: metadata about one of the links in a message
@@ -429,6 +492,7 @@ type JsonSentTranscriptMessage struct {
 	ExpirationStartTimestamp int64             `json:"expirationStartTimestamp,omitempty" yaml:"expirationStartTimestamp,omitempty"`
 	IsRecipientUpdate        bool              `json:"isRecipientUpdate,omitempty" yaml:"isRecipientUpdate,omitempty"`
 	Message                  *JsonDataMessage  `json:"message,omitempty" yaml:"message,omitempty"`
+	Story                    *StoryMessage     `json:"story,omitempty" yaml:"story,omitempty"`
 	Timestamp                int64             `json:"timestamp,omitempty" yaml:"timestamp,omitempty"`
 	UnidentifiedStatus       map[string]string `json:"unidentifiedStatus,omitempty" yaml:"unidentifiedStatus,omitempty"`
 }
@@ -519,6 +583,11 @@ type OfferMessage struct {
 	Type   string `json:"type,omitempty" yaml:"type,omitempty"`
 }
 
+type PagingData struct {
+	HasMorePages     bool  `json:"has_more_pages,omitempty" yaml:"has_more_pages,omitempty"`
+	NextPageRevision int32 `json:"next_page_revision,omitempty" yaml:"next_page_revision,omitempty"`
+}
+
 // Payment: details about a MobileCoin payment
 type Payment struct {
 	Note    string `json:"note,omitempty" yaml:"note,omitempty"`       // note attached to the payment
@@ -532,6 +601,7 @@ type Profile struct {
 	Avatar            string        `json:"avatar,omitempty" yaml:"avatar,omitempty"` // path to avatar on local disk
 	Capabilities      *Capabilities `json:"capabilities,omitempty" yaml:"capabilities,omitempty"`
 	Color             string        `json:"color,omitempty" yaml:"color,omitempty"`               // color of the chat with this user
+	ContactName       string        `json:"contact_name,omitempty" yaml:"contact_name,omitempty"` // The user's name from local contact names
 	Emoji             string        `json:"emoji,omitempty" yaml:"emoji,omitempty"`
 	ExpirationTime    int32         `json:"expiration_time,omitempty" yaml:"expiration_time,omitempty"`
 	InboxPosition     int32         `json:"inbox_position,omitempty" yaml:"inbox_position,omitempty"`
@@ -566,6 +636,7 @@ type ReceiptMessage struct {
 type RefuseMembershipRequest struct {
 	Request
 	Account string         `json:"account,omitempty" yaml:"account,omitempty"` // The account to interact with
+	AlsoBan bool           `json:"also_ban,omitempty" yaml:"also_ban,omitempty"`
 	GroupId string         `json:"group_id,omitempty" yaml:"group_id,omitempty"`
 	Members []*JsonAddress `json:"members,omitempty" yaml:"members,omitempty"` // list of requesting members to refuse
 }
@@ -658,6 +729,7 @@ type SendPaymentRequest struct {
 
 type SendRequest struct {
 	Request
+	Account          string            `json:"account,omitempty" yaml:"account,omitempty"`
 	Attachments      []*JsonAttachment `json:"attachments,omitempty" yaml:"attachments,omitempty"`
 	Members          []*JsonAddress    `json:"members,omitempty" yaml:"members,omitempty"` // Optionally set to a sub-set of group members. Ignored if recipientGroupId isn't specified
 	Mentions         []*JsonMention    `json:"mentions,omitempty" yaml:"mentions,omitempty"`
@@ -682,6 +754,14 @@ type SendSuccess struct {
 	Unidentified bool    `json:"unidentified,omitempty" yaml:"unidentified,omitempty"`
 }
 
+// SendSyncMessageRequest: Sends a sync message to the account's devices
+type SendSyncMessageRequest struct {
+	Request
+	Account                string                             `json:"account,omitempty" yaml:"account,omitempty"`
+	MessageRequestResponse *JsonMessageRequestResponseMessage `json:"message_request_response,omitempty" yaml:"message_request_response,omitempty"` // This can be set to indicate to other devices about a response to an incoming message request from an unknown user or group. Warning: Using the BLOCK and BLOCK_AND_DELETE options relies on other devices to do the blocking, and it does not make you leave the group!
+	ViewOnceOpenMessage    *JsonViewOnceOpenMessage           `json:"view_once_open_message,omitempty" yaml:"view_once_open_message,omitempty"`     // This can be set to indicate to other devices about having viewed a view-once message.
+}
+
 // Server: a Signal server
 type Server struct {
 	Ca                     string       `json:"ca,omitempty" yaml:"ca,omitempty"` // base64 encoded trust store, password must be 'whisper'
@@ -784,6 +864,23 @@ type SharedContactPhone struct {
 	Value string `json:"value,omitempty" yaml:"value,omitempty"` // the phone number
 }
 
+// StorageChange: Broadcast to subscribed clients when there is a state change from the storage service
+type StorageChange struct {
+	Version int64 `json:"version,omitempty" yaml:"version,omitempty"` // Seems to behave like the group version numbers and increments every time the state changes
+}
+
+type StoryContext struct {
+	Author        string `json:"author,omitempty" yaml:"author,omitempty"`
+	SentTimestamp int64  `json:"sent_timestamp,omitempty" yaml:"sent_timestamp,omitempty"`
+}
+
+type StoryMessage struct {
+	AllowReplies bool             `json:"allow_replies,omitempty" yaml:"allow_replies,omitempty"`
+	File         *JsonAttachment  `json:"file,omitempty" yaml:"file,omitempty"`
+	Group        *JsonGroupV2Info `json:"group,omitempty" yaml:"group,omitempty"`
+	Text         *TextAttachment  `json:"text,omitempty" yaml:"text,omitempty"`
+}
+
 type SubmitChallengeRequest struct {
 	Request
 	Account      string `json:"account,omitempty" yaml:"account,omitempty"`
@@ -797,6 +894,16 @@ type SubscribeRequest struct {
 	Account string `json:"account,omitempty" yaml:"account,omitempty"` // The account to subscribe to incoming message for
 }
 
+type TextAttachment struct {
+	BackgroundColor     string       `json:"background_color,omitempty" yaml:"background_color,omitempty"`
+	BackgroundGradient  *Gradient    `json:"background_gradient,omitempty" yaml:"background_gradient,omitempty"`
+	Preview             *JsonPreview `json:"preview,omitempty" yaml:"preview,omitempty"`
+	Style               string       `json:"style,omitempty" yaml:"style,omitempty"`
+	Text                string       `json:"text,omitempty" yaml:"text,omitempty"`
+	TextBackgroundColor string       `json:"text_background_color,omitempty" yaml:"text_background_color,omitempty"`
+	TextForegroundColor string       `json:"text_foreground_color,omitempty" yaml:"text_foreground_color,omitempty"`
+}
+
 // TrustRequest: Trust another user's safety number using either the QR code data or the safety number text
 type TrustRequest struct {
 	Request
@@ -823,6 +930,14 @@ type TypingRequest struct {
 	When    int64        `json:"when,omitempty" yaml:"when,omitempty"`
 }
 
+// UnbanUserRequest: Unbans users from a group.
+type UnbanUserRequest struct {
+	Request
+	Account string         `json:"account,omitempty" yaml:"account,omitempty"` // The account to interact with
+	GroupId string         `json:"group_id,omitempty" yaml:"group_id,omitempty"`
+	Users   []*JsonAddress `json:"users,omitempty" yaml:"users,omitempty"` // List of users to unban
+}
+
 // UnsubscribeRequest: See subscribe for more info
 type UnsubscribeRequest struct {
 	Request

signald/signald.go

@@ -99,8 +99,12 @@ func (s *Signald) connect() error {
 	return nil
 }
 
+func (s *Signald) Close() error {
+	return s.socket.Close()
+}
+
 // Listen listens for events from signald
-func (s *Signald) Listen(c chan client_protocol.BasicResponse) {
+func (s *Signald) Listen(c chan client_protocol.BasicResponse) error {
 	for {
 		msg, err := s.readNext()
 		if err == io.EOF {
@@ -108,7 +112,11 @@ func (s *Signald) Listen(c chan client_protocol.BasicResponse) {
 			if c != nil {
 				close(c)
 			}
-			return
+			return nil
+		}
+
+		if err != nil {
+			return err
 		}
 
 		if msg.Type == "unexpected_error" {
@@ -125,7 +133,7 @@ func (s *Signald) Listen(c chan client_protocol.BasicResponse) {
 			subscribers <- msg
 		}
 
-		if c != nil {
+		if c != nil && !(msg.ID == "" && msg.Type == "version") {
 			c <- msg
 		}
 	}
@@ -170,9 +178,6 @@ func (s *Signald) readNext() (b client_protocol.BasicResponse, err error) {
 	} else {
 		err = json.NewDecoder(s.socket).Decode(&b)
 	}
-	if err != nil {
+
-		log.Println("signald-go: error decoding message from signald:", err)
-		return
-	}
 	return
 }

tools/generator/requests.go.tmpl

@@ -22,15 +22,14 @@ func {{$action.FnName}}(conn *signald.Signald) ({{if ne $action.Response ""}}res
   if(r.ID == "") {
     r.ID = signald.GenerateID()
   }
+  responseChannel := conn.GetResponseListener(r.ID)
+  defer conn.CloseResponseListener(r.ID)
   err = conn.RawRequest(r)
   if err != nil {
     log.Println("signald-go: error submitting request to signald")
     return
   }
 
-  responseChannel := conn.GetResponseListener(r.ID)
-  defer conn.CloseResponseListener(r.ID)
-
   rawResponse := <- responseChannel
 	if rawResponse.Error != nil {
 		err = mkerr(rawResponse)