diff --git a/cmd/config-generator/config_struct.go b/cmd/config-generator/config_struct.go new file mode 100644 index 0000000..cc2ed5b --- /dev/null +++ b/cmd/config-generator/config_struct.go @@ -0,0 +1,173 @@ +package main + +type WhisperServerConfiguration struct { + Twilio TwilioConfiguration `yaml:"twilio"` + Push PushConfiguration `yaml:"push"` + AWSAttachments S3Configuration `yaml:"awsAttachments"` + GCPAttachments GCPAttachmentsConfiguration `yaml:"gcpAttachments"` + CDN S3Configuration `yaml:"cdn"` + Cache RedisConfiguration `yaml:"cache"` + Pubsub RedisConfiguration `yaml:"pubsub"` + Directory DirectoryConfiguration `yaml:"directory"` + AccountDatabaseCrawler AccountDatabaseCrawlerConfiguration `yaml:"accountDatabaseCrawler"` + PushScheduler RedisConfiguration `yaml:"pushScheduler"` + MessageCache MessageCacheConfiguration `yaml:"messageCache"` + MessageStore DatabaseConfiguration `yaml:"messageStore"` + AbuseDatabase DatabaseConfiguration `yaml:"abuseDatabase"` + TestDevices []TestDeviceConfiguration `yaml:"testDevices"` + MaxDevices []MaxDeviceConfiguration `yaml:"maxDevices"` + AccountsDatabase DatabaseConfiguration `yaml:"accountsDatabase"` + // Limits RateLimitsConfiguration `yaml:"limits"` + // HTTPClient JerseyClientConfiguration `yaml:"httpClient"` + // WebSocket WebSocketConfiguration `yaml:"webSocket"` + Turn TurnConfiguration `yaml:"turn"` + GCM GCMConfiguration `yaml:"gcm"` + APN APNConfiguration `yaml:"apn"` + UnidentifiedDelivery UnidentifiedDeliveryConfiguration `yaml:"unidentifiedDelivery"` + VoiceVerification VoiceVerificationConfiguration `yaml:"voiceVerification"` + Recaptcha RecaptchaConfiguration `yaml:"recaptcha"` + StorageService SecureStorageServiceConfiguration `yaml:"storageService"` + BackupService SecureBackupServiceConfiguration `yaml:"backupService"` + ZKConfig ZKConfig `yaml:"zkConfig"` + RemoteConfig RemoteConfigConfiguration `yaml:"remoteConfig"` + TransparentDataIndex map[string]string `yaml:"transparentDataIndex"` +} + +type TwilioConfiguration struct { + AccountID string `yaml:"accountId"` + AccountToken string `yaml:"accountToken"` + Numbers []string `yaml:"numbers"` + LocalDomain string `yaml:"localDomain"` + MessagingServicesID string `yaml:"messagingServicesId"` + // CircuitBreaker CircuitBreakerConfiguration `yaml:"circuitBreaker"` + // Retry RetryConfiguration `yaml:"retry"` +} + +type PushConfiguration struct { + QueueSize int `yaml:"queueSize"` +} + +type S3Configuration struct { + AccessKey string `yaml:"accessKey"` + AccessSecret string `yaml:"accessSecret"` + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` +} + +type GCPAttachmentsConfiguration struct { + Domain string `yaml:"domain"` + Email string `yaml:"email"` + MaxSizeInBytes uint64 `yaml:"maxSizeInBytes"` + PathPrefix string `yaml:"pathPrefix"` + RSASigningKey string `yaml:"rsaSigningKey"` +} + +type RedisConfiguration struct { + URL string `yaml:"url"` + ReplicaURLs []string `yaml:"replicaUrls"` + // CircuitBreaker CircuitBreakerConfiguration `yaml:"circuitBreaker"` +} + +type DirectoryConfiguration struct { + Redis RedisConfiguration `yaml:"redis"` + SQS SQSConfiguration `yaml:"sqs"` + Client DirectoryClientConfiguration `yaml:"client"` + Server DirectoryServerConfiguration `yaml:"server"` +} + +type AccountDatabaseCrawlerConfiguration struct { + ChunkSize uint64 `yaml:"chunkSize"` + ChunkIntervalMS uint64 `yaml:"chunkIntervalMs"` +} + +type MessageCacheConfiguration struct { + Redis RedisConfiguration `yaml:"redis"` + PersistDelayMinutes int `yaml:"persistDelayMinutes"` +} + +type DatabaseConfiguration struct { + // CircuitBreaker CircuitBreakerConfiguration `yaml:"circuitBreaker"` + DriverClass string `yaml:"driverClass"` + Password string `yaml:"password"` + URL string `yaml:"url"` + User string `yaml:"user"` +} + +type TestDeviceConfiguration struct { + Number string `yaml:"number"` + Code int `yaml:"code"` +} + +type MaxDeviceConfiguration struct { + Number string `yaml:"number"` + Count int `yaml:"count"` +} + +type TurnConfiguration struct { + Secret string `yaml:"secret"` + URIs []string `yaml:"uris"` +} + +type GCMConfiguration struct { + SenderID int64 `yaml:"senderId"` + APIKey string `yaml:"apiKey"` +} + +type APNConfiguration struct { + PushCertificate string `yaml:"pushCertificate"` + PushKey string `yaml:"pushKey"` + BundleID string `yaml:"bundleId"` +} + +type UnidentifiedDeliveryConfiguration struct { + Certificate string `yaml:"certificate"` + PrivateKey string `yaml:"privateKey"` + ExpiresDays int `yaml:"expiresDays"` +} + +type VoiceVerificationConfiguration struct { + URL string `yaml:"url"` + Locales []string `yaml:"locales"` +} + +type RecaptchaConfiguration struct { + Secret string `yaml:"secret"` +} + +type SecureStorageServiceConfiguration struct { + UserAuthenticationTokenSharedSecret string `yaml:"userAuthenticationTokenSharedSecret"` +} + +type SecureBackupServiceConfiguration struct { + UserAuthenticationTokenSharedSecret string `yaml:"userAuthenticationTokenSharedSecret"` +} + +type ZKConfig struct { + ServerSecret string `yaml:"serverSecret"` + ServerPublic string `yaml:"serverPublic"` + Enabled bool `yaml:"enabled"` +} + +type RemoteConfigConfiguration struct { + AuthorizedTokens []string `yaml:"authorizedTokens"` +} + +type SQSConfiguration struct { + AccessKey string `yaml:"accessKey"` + AccessSecret string `yaml:"accessSecret"` + QueueURL string `yaml:"queueUrl"` +} + +// DirectoryClientConfiguration is for interfacing with Contact Discovery Service cluster +type DirectoryClientConfiguration struct { + // UserAuthenticationTokenSharedSecret is a hex-encoded secret shared with CDS used to generate auth tokens for Signal users + UserAuthenticationTokenSharedSecret string `yaml:"userAuthenticationTokenSharedSecret"` + // UserAuthenticationTokenUserIDSecret is a hex-encoded secret shared among Signal-Servers to obscure user phone numbers from CDS + UserAuthenticationTokenUserIDSecret string `yaml:"userAuthenticationTokenUserIdSecret"` +} + +type DirectoryServerConfiguration struct { + ReplicationURL string `yaml:"replicationUrl"` + ReplicationPassword string `yaml:"replicationPassword"` + ReplicationCACertificate string `yaml:"replicationCaCertificate"` +} diff --git a/cmd/config-generator/main.go b/cmd/config-generator/main.go new file mode 100644 index 0000000..f07d526 --- /dev/null +++ b/cmd/config-generator/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "io" + "log" + "os" + + "github.com/c2h5oh/datasize" + "gopkg.in/yaml.v2" +) + +var ( + // configFilename = "/etc/signal-server/config.yaml" + caCertificate string + defaultRedisConfig = RedisConfiguration{ + URL: "redis://redis:6379/", + ReplicaURLs: []string{"redis://redis:6379/"}, + } + defaultPostgresConfig = DatabaseConfiguration{ + DriverClass: "org.postgresql.Driver", + Password: "password", + URL: "jdbc:postgresql://db/signal?user=signal&password=password", + User: "signal", + } +) + +func main() { + config := WhisperServerConfiguration{ + Twilio: TwilioConfiguration{ + AccountID: "a", + AccountToken: "a", + LocalDomain: "signal-server.invalid", + MessagingServicesID: "a", + Numbers: []string{"+12024561414"}, + }, + Push: PushConfiguration{QueueSize: 100}, + AWSAttachments: S3Configuration{ + AccessKey: "a", + AccessSecret: "a", + Bucket: "attachments", + Region: "us-fake-0", + }, + GCPAttachments: GCPAttachmentsConfiguration{ + Domain: "signal-server.invalid", + Email: "fake@signal-server.invalid", + MaxSizeInBytes: 500 * datasize.MB.Bytes(), + RSASigningKey: GenerateGCPSigningKey(), + }, + CDN: S3Configuration{ + AccessKey: "a", + AccessSecret: "a", + Bucket: "cdn", + Region: "us-fake-0", + }, + Cache: defaultRedisConfig, + Pubsub: defaultRedisConfig, + Directory: DirectoryConfiguration{ + Redis: defaultRedisConfig, + SQS: SQSConfiguration{ + AccessKey: "a", + AccessSecret: "a", + QueueURL: "a", + }, + Client: DirectoryClientConfiguration{ + UserAuthenticationTokenSharedSecret: "00", + UserAuthenticationTokenUserIDSecret: "00", + }, + Server: DirectoryServerConfiguration{ + ReplicationURL: "https://fake.invalid/TODO", + ReplicationPassword: "a", + ReplicationCACertificate: DirectoryReplicationServerCA(), + }, + }, + AccountDatabaseCrawler: AccountDatabaseCrawlerConfiguration{ + ChunkSize: datasize.MB.Bytes(), + ChunkIntervalMS: 1000, + }, + PushScheduler: defaultRedisConfig, + MessageCache: MessageCacheConfiguration{ + Redis: defaultRedisConfig, + PersistDelayMinutes: 60, + }, + MessageStore: defaultPostgresConfig, + AbuseDatabase: defaultPostgresConfig, + TestDevices: []TestDeviceConfiguration{}, + MaxDevices: []MaxDeviceConfiguration{}, + AccountsDatabase: defaultPostgresConfig, + Turn: TurnConfiguration{ + Secret: "a", + URIs: []string{"stun:fake.invalid:80"}, + }, + GCM: GCMConfiguration{ + SenderID: 0, + APIKey: "fake.invalid", + }, + APN: GenerateAPNConfiguration(), + UnidentifiedDelivery: UnidentifiedDeliveryConfiguration{ + Certificate: "aaaa", + PrivateKey: "YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=", + ExpiresDays: 90, + }, + VoiceVerification: VoiceVerificationConfiguration{ + URL: "https://fake.invalid/voice", + Locales: []string{"en"}, + }, + Recaptcha: RecaptchaConfiguration{Secret: "a"}, + StorageService: SecureStorageServiceConfiguration{UserAuthenticationTokenSharedSecret: "00"}, + BackupService: SecureBackupServiceConfiguration{UserAuthenticationTokenSharedSecret: "00"}, + ZKConfig: GenerateZKConfig(), + RemoteConfig: RemoteConfigConfiguration{ + AuthorizedTokens: []string{"a"}, + }, + } + + var writer io.Writer + + writer = os.Stdout + + if len(os.Args) > 1 { + log.Printf("Writing config to %s", os.Args[1]) + var err error + f, err := os.Create(os.Args[1]) + if err != nil { + panic(err) + } + defer f.Close() + writer = f + } + + err := yaml.NewEncoder(writer).Encode(config) + if err != nil { + panic(err) + } +} diff --git a/cmd/config-generator/pki_generators.go b/cmd/config-generator/pki_generators.go new file mode 100644 index 0000000..31726ae --- /dev/null +++ b/cmd/config-generator/pki_generators.go @@ -0,0 +1,116 @@ +package main + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "log" + "math/big" + "time" +) + +func createCertPEM(template, parent *x509.Certificate, pub, priv interface{}) string { + caCertBytes, err := x509.CreateCertificate(rand.Reader, template, parent, pub, priv) + if err != nil { + log.Println("failed to generate CA certificate") + panic(err) + } + + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCertBytes})) +} + +func keyToPem(key *rsa.PrivateKey) string { + pkcs1 := x509.MarshalPKCS1PrivateKey(key) + block := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: pkcs1, + } + encoded := pem.EncodeToMemory(block) + return string(encoded) +} + +func DirectoryReplicationServerCA() string { + ca := &x509.Certificate{ + SerialNumber: big.NewInt(1653), + Subject: pkix.Name{ + Organization: []string{"ORGANIZATION_NAME"}, + Country: []string{"COUNTRY_CODE"}, + Province: []string{"PROVINCE"}, + Locality: []string{"CITY"}, + StreetAddress: []string{"ADDRESS"}, + PostalCode: []string{"POSTAL_CODE"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caPrivateKey, _ := rsa.GenerateKey(rand.Reader, 2048) + return createCertPEM(ca, ca, &caPrivateKey.PublicKey, caPrivateKey) +} + +func GenerateAPNConfiguration() (a APNConfiguration) { + ca := &x509.Certificate{ + SerialNumber: big.NewInt(1653), + Subject: pkix.Name{ + Organization: []string{"ORGANIZATION_NAME"}, + Country: []string{"COUNTRY_CODE"}, + Province: []string{"PROVINCE"}, + Locality: []string{"CITY"}, + StreetAddress: []string{"ADDRESS"}, + PostalCode: []string{"POSTAL_CODE"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + caPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + log.Println("Failied to generate 2048 bit RSA key for APN CA") + panic(err) + } + + caPem := createCertPEM(ca, ca, &caPrivateKey.PublicKey, caPrivateKey) + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + log.Println("Failied to generate 2048 bit RSA key for APN configuration") + panic(err) + } + + certPem := createCertPEM(&x509.Certificate{ + SerialNumber: big.NewInt(1653), + Subject: pkix.Name{ + Organization: []string{"ORGANIZATION_NAME"}, + Country: []string{"COUNTRY_CODE"}, + Province: []string{"PROVINCE"}, + Locality: []string{"CITY"}, + StreetAddress: []string{"ADDRESS"}, + PostalCode: []string{"POSTAL_CODE"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + KeyUsage: x509.KeyUsageDigitalSignature, + }, ca, &privateKey.PublicKey, privateKey) + + a.PushCertificate = caPem + certPem + a.PushKey = keyToPem(privateKey) + a.BundleID = "a" + return +} + +func GenerateGCPSigningKey() string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + panic(err) + } + return keyToPem(key) +} diff --git a/cmd/config-generator/zkgroups.go b/cmd/config-generator/zkgroups.go new file mode 100644 index 0000000..017e8bf --- /dev/null +++ b/cmd/config-generator/zkgroups.go @@ -0,0 +1,37 @@ +package main + +import ( + "bytes" + "os" + "os/exec" + "strings" +) + +func GenerateZKConfig() (z ZKConfig) { + z.Enabled = false + + cmd := exec.Command("java", "-jar", "/usr/share/TextSecureServer.jar", "zkparams") + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + panic(err) + } + for _, line := range strings.Split(out.String(), "\n") { + if len(line) == 0 { + continue + } + parts := strings.Split(line, ": ") + if len(parts) != 2 { + continue + } else { + } + if parts[0] == "Public" { + z.ServerPublic = parts[1] + } else if parts[0] == "Private" { + z.ServerSecret = parts[1] + } + } + return +} diff --git a/testhelper.go b/cmd/test-helper/main.go similarity index 100% rename from testhelper.go rename to cmd/test-helper/main.go diff --git a/docker-compose.yml b/docker-compose.yml index 2cff315..8f188c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,30 +13,26 @@ services: POSTGRES_USER: signal expose: - 5432 - volumes: - - "./postgres-data:/var/lib/postgresql/data" adminer: image: adminer restart: always ports: - "127.0.0.1:8083:8080" - testhelper: - image: registry.git.callpipe.com/finn/signald-test-infrastructure/testhelper:master + test-helper: + image: registry.gitlab.com/signald/test-infrastructure/test-helper:master restart: always build: context: . - dockerfile: testhelper.Dockerfile + dockerfile: test-helper.Dockerfile environment: DB: "host=db user=signal dbname=signal password=password sslmode=disable" ports: - "127.0.0.1:8082:8082" signal: - image: registry.git.callpipe.com/finn/signald-test-infrastructure/signal-server:2.92 + image: registry.gitlab.com/signald/test-infrastructure/signal-server:3.21 build: context: . dockerfile: signal-server.Dockerfile - volumes: - - "./signal-server.yaml:/etc/signal-server.yaml" ports: - "127.0.0.1:8080:8080" - "127.0.0.1:8081:8081" diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..277f8e8 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module gitlab.com/signald/test-infrastructure + +go 1.14 + +require ( + github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee + github.com/lib/pq v1.8.0 + gopkg.in/yaml.v2 v2.3.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f04e381 --- /dev/null +++ b/go.sum @@ -0,0 +1,8 @@ +github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee h1:BnPxIde0gjtTnc9Er7cxvBk8DHLWhEux0SxayC8dP6I= +github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= +github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= +github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/migrate-and-start-server.sh b/migrate-and-start-server.sh index cd97ec3..434f7fd 100755 --- a/migrate-and-start-server.sh +++ b/migrate-and-start-server.sh @@ -1,7 +1,13 @@ #!/bin/bash set -exu -for db in abusedb keysdb accountdb messagedb; do + +CONFIG_FILE="/etc/signal-server/config.yaml" + +# generate config +/usr/bin/config-generator | tee "${CONFIG_FILE}" + +for db in abusedb accountdb messagedb; do echo "Migrating $db" - java -jar /usr/share/TextSecureServer.jar "$db" migrate /etc/signal-server.yaml + java -jar /usr/share/TextSecureServer.jar "$db" migrate "${CONFIG_FILE}" done -java -jar /usr/share/TextSecureServer.jar server /etc/signal-server.yaml +java -jar /usr/share/TextSecureServer.jar server "${CONFIG_FILE}" diff --git a/signal-server.Dockerfile b/signal-server.Dockerfile index caaa4fb..fe4f504 100644 --- a/signal-server.Dockerfile +++ b/signal-server.Dockerfile @@ -1,5 +1,13 @@ +FROM golang:latest as golang-build +COPY go.mod go.sum /go/src/gitlab.com/signald/test-infrastructure/ +COPY cmd/ /go/src/gitlab.com/signald/test-infrastructure/cmd +WORKDIR /go/src/gitlab.com/signald/test-infrastructure/ +RUN go mod download +RUN go build ./cmd/config-generator + FROM debian:buster as build -RUN apt-get update && apt-get install -y openjdk-11-jdk-headless maven git +RUN apt-get update && apt-get install -y openjdk-11-jre-headless +RUN apt-get install -y openjdk-11-jdk-headless maven git RUN git -C /usr/local/src clone https://github.com/signalapp/Signal-Server WORKDIR /usr/local/src/Signal-Server COPY signal-server-patches /tmp/signal-server-patches @@ -8,9 +16,11 @@ RUN mvn install -DskipTests FROM debian:buster RUN apt-get update && apt-get install -y openjdk-11-jre-headless -COPY --from=build /usr/local/src/Signal-Server/service/target/TextSecureServer-2.92.jar /usr/share/TextSecureServer.jar +COPY --from=build /usr/local/src/Signal-Server/service/target/TextSecureServer-*.jar /usr/share/TextSecureServer.jar +COPY --from=golang-build /go/src/gitlab.com/signald/test-infrastructure/config-generator /usr/bin/config-generator COPY migrate-and-start-server.sh /usr/bin/migrate-and-start-server +RUN mkdir -p /etc/signal-server RUN useradd signal -RUN chown -R signal /usr/share/TextSecureServer.jar +RUN chown -R signal /usr/share/TextSecureServer.jar /etc/signal-server USER signal CMD ["/usr/bin/migrate-and-start-server"] diff --git a/test-helper.Dockerfile b/test-helper.Dockerfile new file mode 100644 index 0000000..ddd714a --- /dev/null +++ b/test-helper.Dockerfile @@ -0,0 +1,10 @@ +FROM golang:latest as golang-build +COPY go.mod go.sum /go/src/gitlab.com/signald/test-infrastructure/ +COPY cmd/ /go/src/gitlab.com/signald/test-infrastructure/cmd +WORKDIR /go/src/gitlab.com/signald/test-infrastructure/ +RUN go mod download +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ./cmd/test-helper + +FROM scratch +COPY --from=golang-build /go/src/gitlab.com/signald/test-infrastructure/test-helper /test-helper +ENTRYPOINT ["/test-helper"] diff --git a/testhelper.Dockerfile b/testhelper.Dockerfile deleted file mode 100644 index edd5951..0000000 --- a/testhelper.Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM golang:latest as build -COPY testhelper.go /go/testhelper.go -RUN go get -u github.com/lib/pq -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build testhelper.go - -FROM scratch -COPY --from=build /go/testhelper /testhelper -ENTRYPOINT ["/testhelper"]