Add tooling to generate a full config on the fly, doesn't quite work yet

This commit is contained in:
Finn 2020-08-16 20:54:50 -07:00
parent f6c915bc30
commit 84def01d68
12 changed files with 513 additions and 22 deletions

View file

@ -0,0 +1,173 @@
package main
type WhisperServerConfiguration struct {
Twilio TwilioConfiguration `yaml:"twilio"`
Push PushConfiguration `yaml:"push"`
AWSAttachments S3Configuration `yaml:"awsAttachments"`
GCPAttachments GCPAttachmentsConfiguration `yaml:"gcpAttachments"`
CDN S3Configuration `yaml:"cdn"`
Cache RedisConfiguration `yaml:"cache"`
Pubsub RedisConfiguration `yaml:"pubsub"`
Directory DirectoryConfiguration `yaml:"directory"`
AccountDatabaseCrawler AccountDatabaseCrawlerConfiguration `yaml:"accountDatabaseCrawler"`
PushScheduler RedisConfiguration `yaml:"pushScheduler"`
MessageCache MessageCacheConfiguration `yaml:"messageCache"`
MessageStore DatabaseConfiguration `yaml:"messageStore"`
AbuseDatabase DatabaseConfiguration `yaml:"abuseDatabase"`
TestDevices []TestDeviceConfiguration `yaml:"testDevices"`
MaxDevices []MaxDeviceConfiguration `yaml:"maxDevices"`
AccountsDatabase DatabaseConfiguration `yaml:"accountsDatabase"`
// Limits RateLimitsConfiguration `yaml:"limits"`
// HTTPClient JerseyClientConfiguration `yaml:"httpClient"`
// WebSocket WebSocketConfiguration `yaml:"webSocket"`
Turn TurnConfiguration `yaml:"turn"`
GCM GCMConfiguration `yaml:"gcm"`
APN APNConfiguration `yaml:"apn"`
UnidentifiedDelivery UnidentifiedDeliveryConfiguration `yaml:"unidentifiedDelivery"`
VoiceVerification VoiceVerificationConfiguration `yaml:"voiceVerification"`
Recaptcha RecaptchaConfiguration `yaml:"recaptcha"`
StorageService SecureStorageServiceConfiguration `yaml:"storageService"`
BackupService SecureBackupServiceConfiguration `yaml:"backupService"`
ZKConfig ZKConfig `yaml:"zkConfig"`
RemoteConfig RemoteConfigConfiguration `yaml:"remoteConfig"`
TransparentDataIndex map[string]string `yaml:"transparentDataIndex"`
}
type TwilioConfiguration struct {
AccountID string `yaml:"accountId"`
AccountToken string `yaml:"accountToken"`
Numbers []string `yaml:"numbers"`
LocalDomain string `yaml:"localDomain"`
MessagingServicesID string `yaml:"messagingServicesId"`
// CircuitBreaker CircuitBreakerConfiguration `yaml:"circuitBreaker"`
// Retry RetryConfiguration `yaml:"retry"`
}
type PushConfiguration struct {
QueueSize int `yaml:"queueSize"`
}
type S3Configuration struct {
AccessKey string `yaml:"accessKey"`
AccessSecret string `yaml:"accessSecret"`
Bucket string `yaml:"bucket"`
Region string `yaml:"region"`
}
type GCPAttachmentsConfiguration struct {
Domain string `yaml:"domain"`
Email string `yaml:"email"`
MaxSizeInBytes uint64 `yaml:"maxSizeInBytes"`
PathPrefix string `yaml:"pathPrefix"`
RSASigningKey string `yaml:"rsaSigningKey"`
}
type RedisConfiguration struct {
URL string `yaml:"url"`
ReplicaURLs []string `yaml:"replicaUrls"`
// CircuitBreaker CircuitBreakerConfiguration `yaml:"circuitBreaker"`
}
type DirectoryConfiguration struct {
Redis RedisConfiguration `yaml:"redis"`
SQS SQSConfiguration `yaml:"sqs"`
Client DirectoryClientConfiguration `yaml:"client"`
Server DirectoryServerConfiguration `yaml:"server"`
}
type AccountDatabaseCrawlerConfiguration struct {
ChunkSize uint64 `yaml:"chunkSize"`
ChunkIntervalMS uint64 `yaml:"chunkIntervalMs"`
}
type MessageCacheConfiguration struct {
Redis RedisConfiguration `yaml:"redis"`
PersistDelayMinutes int `yaml:"persistDelayMinutes"`
}
type DatabaseConfiguration struct {
// CircuitBreaker CircuitBreakerConfiguration `yaml:"circuitBreaker"`
DriverClass string `yaml:"driverClass"`
Password string `yaml:"password"`
URL string `yaml:"url"`
User string `yaml:"user"`
}
type TestDeviceConfiguration struct {
Number string `yaml:"number"`
Code int `yaml:"code"`
}
type MaxDeviceConfiguration struct {
Number string `yaml:"number"`
Count int `yaml:"count"`
}
type TurnConfiguration struct {
Secret string `yaml:"secret"`
URIs []string `yaml:"uris"`
}
type GCMConfiguration struct {
SenderID int64 `yaml:"senderId"`
APIKey string `yaml:"apiKey"`
}
type APNConfiguration struct {
PushCertificate string `yaml:"pushCertificate"`
PushKey string `yaml:"pushKey"`
BundleID string `yaml:"bundleId"`
}
type UnidentifiedDeliveryConfiguration struct {
Certificate string `yaml:"certificate"`
PrivateKey string `yaml:"privateKey"`
ExpiresDays int `yaml:"expiresDays"`
}
type VoiceVerificationConfiguration struct {
URL string `yaml:"url"`
Locales []string `yaml:"locales"`
}
type RecaptchaConfiguration struct {
Secret string `yaml:"secret"`
}
type SecureStorageServiceConfiguration struct {
UserAuthenticationTokenSharedSecret string `yaml:"userAuthenticationTokenSharedSecret"`
}
type SecureBackupServiceConfiguration struct {
UserAuthenticationTokenSharedSecret string `yaml:"userAuthenticationTokenSharedSecret"`
}
type ZKConfig struct {
ServerSecret string `yaml:"serverSecret"`
ServerPublic string `yaml:"serverPublic"`
Enabled bool `yaml:"enabled"`
}
type RemoteConfigConfiguration struct {
AuthorizedTokens []string `yaml:"authorizedTokens"`
}
type SQSConfiguration struct {
AccessKey string `yaml:"accessKey"`
AccessSecret string `yaml:"accessSecret"`
QueueURL string `yaml:"queueUrl"`
}
// DirectoryClientConfiguration is for interfacing with Contact Discovery Service cluster
type DirectoryClientConfiguration struct {
// UserAuthenticationTokenSharedSecret is a hex-encoded secret shared with CDS used to generate auth tokens for Signal users
UserAuthenticationTokenSharedSecret string `yaml:"userAuthenticationTokenSharedSecret"`
// UserAuthenticationTokenUserIDSecret is a hex-encoded secret shared among Signal-Servers to obscure user phone numbers from CDS
UserAuthenticationTokenUserIDSecret string `yaml:"userAuthenticationTokenUserIdSecret"`
}
type DirectoryServerConfiguration struct {
ReplicationURL string `yaml:"replicationUrl"`
ReplicationPassword string `yaml:"replicationPassword"`
ReplicationCACertificate string `yaml:"replicationCaCertificate"`
}

View file

@ -0,0 +1,134 @@
package main
import (
"io"
"log"
"os"
"github.com/c2h5oh/datasize"
"gopkg.in/yaml.v2"
)
var (
// configFilename = "/etc/signal-server/config.yaml"
caCertificate string
defaultRedisConfig = RedisConfiguration{
URL: "redis://redis:6379/",
ReplicaURLs: []string{"redis://redis:6379/"},
}
defaultPostgresConfig = DatabaseConfiguration{
DriverClass: "org.postgresql.Driver",
Password: "password",
URL: "jdbc:postgresql://db/signal?user=signal&password=password",
User: "signal",
}
)
func main() {
config := WhisperServerConfiguration{
Twilio: TwilioConfiguration{
AccountID: "a",
AccountToken: "a",
LocalDomain: "signal-server.invalid",
MessagingServicesID: "a",
Numbers: []string{"+12024561414"},
},
Push: PushConfiguration{QueueSize: 100},
AWSAttachments: S3Configuration{
AccessKey: "a",
AccessSecret: "a",
Bucket: "attachments",
Region: "us-fake-0",
},
GCPAttachments: GCPAttachmentsConfiguration{
Domain: "signal-server.invalid",
Email: "fake@signal-server.invalid",
MaxSizeInBytes: 500 * datasize.MB.Bytes(),
RSASigningKey: GenerateGCPSigningKey(),
},
CDN: S3Configuration{
AccessKey: "a",
AccessSecret: "a",
Bucket: "cdn",
Region: "us-fake-0",
},
Cache: defaultRedisConfig,
Pubsub: defaultRedisConfig,
Directory: DirectoryConfiguration{
Redis: defaultRedisConfig,
SQS: SQSConfiguration{
AccessKey: "a",
AccessSecret: "a",
QueueURL: "a",
},
Client: DirectoryClientConfiguration{
UserAuthenticationTokenSharedSecret: "00",
UserAuthenticationTokenUserIDSecret: "00",
},
Server: DirectoryServerConfiguration{
ReplicationURL: "https://fake.invalid/TODO",
ReplicationPassword: "a",
ReplicationCACertificate: DirectoryReplicationServerCA(),
},
},
AccountDatabaseCrawler: AccountDatabaseCrawlerConfiguration{
ChunkSize: datasize.MB.Bytes(),
ChunkIntervalMS: 1000,
},
PushScheduler: defaultRedisConfig,
MessageCache: MessageCacheConfiguration{
Redis: defaultRedisConfig,
PersistDelayMinutes: 60,
},
MessageStore: defaultPostgresConfig,
AbuseDatabase: defaultPostgresConfig,
TestDevices: []TestDeviceConfiguration{},
MaxDevices: []MaxDeviceConfiguration{},
AccountsDatabase: defaultPostgresConfig,
Turn: TurnConfiguration{
Secret: "a",
URIs: []string{"stun:fake.invalid:80"},
},
GCM: GCMConfiguration{
SenderID: 0,
APIKey: "fake.invalid",
},
APN: GenerateAPNConfiguration(),
UnidentifiedDelivery: UnidentifiedDeliveryConfiguration{
Certificate: "aaaa",
PrivateKey: "YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=",
ExpiresDays: 90,
},
VoiceVerification: VoiceVerificationConfiguration{
URL: "https://fake.invalid/voice",
Locales: []string{"en"},
},
Recaptcha: RecaptchaConfiguration{Secret: "a"},
StorageService: SecureStorageServiceConfiguration{UserAuthenticationTokenSharedSecret: "00"},
BackupService: SecureBackupServiceConfiguration{UserAuthenticationTokenSharedSecret: "00"},
ZKConfig: GenerateZKConfig(),
RemoteConfig: RemoteConfigConfiguration{
AuthorizedTokens: []string{"a"},
},
}
var writer io.Writer
writer = os.Stdout
if len(os.Args) > 1 {
log.Printf("Writing config to %s", os.Args[1])
var err error
f, err := os.Create(os.Args[1])
if err != nil {
panic(err)
}
defer f.Close()
writer = f
}
err := yaml.NewEncoder(writer).Encode(config)
if err != nil {
panic(err)
}
}

View file

@ -0,0 +1,116 @@
package main
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"log"
"math/big"
"time"
)
func createCertPEM(template, parent *x509.Certificate, pub, priv interface{}) string {
caCertBytes, err := x509.CreateCertificate(rand.Reader, template, parent, pub, priv)
if err != nil {
log.Println("failed to generate CA certificate")
panic(err)
}
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCertBytes}))
}
func keyToPem(key *rsa.PrivateKey) string {
pkcs1 := x509.MarshalPKCS1PrivateKey(key)
block := &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: pkcs1,
}
encoded := pem.EncodeToMemory(block)
return string(encoded)
}
func DirectoryReplicationServerCA() string {
ca := &x509.Certificate{
SerialNumber: big.NewInt(1653),
Subject: pkix.Name{
Organization: []string{"ORGANIZATION_NAME"},
Country: []string{"COUNTRY_CODE"},
Province: []string{"PROVINCE"},
Locality: []string{"CITY"},
StreetAddress: []string{"ADDRESS"},
PostalCode: []string{"POSTAL_CODE"},
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(10, 0, 0),
IsCA: true,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
}
caPrivateKey, _ := rsa.GenerateKey(rand.Reader, 2048)
return createCertPEM(ca, ca, &caPrivateKey.PublicKey, caPrivateKey)
}
func GenerateAPNConfiguration() (a APNConfiguration) {
ca := &x509.Certificate{
SerialNumber: big.NewInt(1653),
Subject: pkix.Name{
Organization: []string{"ORGANIZATION_NAME"},
Country: []string{"COUNTRY_CODE"},
Province: []string{"PROVINCE"},
Locality: []string{"CITY"},
StreetAddress: []string{"ADDRESS"},
PostalCode: []string{"POSTAL_CODE"},
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(10, 0, 0),
IsCA: true,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
}
caPrivateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
log.Println("Failied to generate 2048 bit RSA key for APN CA")
panic(err)
}
caPem := createCertPEM(ca, ca, &caPrivateKey.PublicKey, caPrivateKey)
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
log.Println("Failied to generate 2048 bit RSA key for APN configuration")
panic(err)
}
certPem := createCertPEM(&x509.Certificate{
SerialNumber: big.NewInt(1653),
Subject: pkix.Name{
Organization: []string{"ORGANIZATION_NAME"},
Country: []string{"COUNTRY_CODE"},
Province: []string{"PROVINCE"},
Locality: []string{"CITY"},
StreetAddress: []string{"ADDRESS"},
PostalCode: []string{"POSTAL_CODE"},
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(10, 0, 0),
KeyUsage: x509.KeyUsageDigitalSignature,
}, ca, &privateKey.PublicKey, privateKey)
a.PushCertificate = caPem + certPem
a.PushKey = keyToPem(privateKey)
a.BundleID = "a"
return
}
func GenerateGCPSigningKey() string {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(err)
}
return keyToPem(key)
}

View file

@ -0,0 +1,37 @@
package main
import (
"bytes"
"os"
"os/exec"
"strings"
)
func GenerateZKConfig() (z ZKConfig) {
z.Enabled = false
cmd := exec.Command("java", "-jar", "/usr/share/TextSecureServer.jar", "zkparams")
var out bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
panic(err)
}
for _, line := range strings.Split(out.String(), "\n") {
if len(line) == 0 {
continue
}
parts := strings.Split(line, ": ")
if len(parts) != 2 {
continue
} else {
}
if parts[0] == "Public" {
z.ServerPublic = parts[1]
} else if parts[0] == "Private" {
z.ServerSecret = parts[1]
}
}
return
}

View file

@ -13,30 +13,26 @@ services:
POSTGRES_USER: signal
expose:
- 5432
volumes:
- "./postgres-data:/var/lib/postgresql/data"
adminer:
image: adminer
restart: always
ports:
- "127.0.0.1:8083:8080"
testhelper:
image: registry.git.callpipe.com/finn/signald-test-infrastructure/testhelper:master
test-helper:
image: registry.gitlab.com/signald/test-infrastructure/test-helper:master
restart: always
build:
context: .
dockerfile: testhelper.Dockerfile
dockerfile: test-helper.Dockerfile
environment:
DB: "host=db user=signal dbname=signal password=password sslmode=disable"
ports:
- "127.0.0.1:8082:8082"
signal:
image: registry.git.callpipe.com/finn/signald-test-infrastructure/signal-server:2.92
image: registry.gitlab.com/signald/test-infrastructure/signal-server:3.21
build:
context: .
dockerfile: signal-server.Dockerfile
volumes:
- "./signal-server.yaml:/etc/signal-server.yaml"
ports:
- "127.0.0.1:8080:8080"
- "127.0.0.1:8081:8081"

9
go.mod Normal file
View file

@ -0,0 +1,9 @@
module gitlab.com/signald/test-infrastructure
go 1.14
require (
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee
github.com/lib/pq v1.8.0
gopkg.in/yaml.v2 v2.3.0
)

8
go.sum Normal file
View file

@ -0,0 +1,8 @@
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee h1:BnPxIde0gjtTnc9Er7cxvBk8DHLWhEux0SxayC8dP6I=
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View file

@ -1,7 +1,13 @@
#!/bin/bash
set -exu
for db in abusedb keysdb accountdb messagedb; do
CONFIG_FILE="/etc/signal-server/config.yaml"
# generate config
/usr/bin/config-generator | tee "${CONFIG_FILE}"
for db in abusedb accountdb messagedb; do
echo "Migrating $db"
java -jar /usr/share/TextSecureServer.jar "$db" migrate /etc/signal-server.yaml
java -jar /usr/share/TextSecureServer.jar "$db" migrate "${CONFIG_FILE}"
done
java -jar /usr/share/TextSecureServer.jar server /etc/signal-server.yaml
java -jar /usr/share/TextSecureServer.jar server "${CONFIG_FILE}"

View file

@ -1,5 +1,13 @@
FROM golang:latest as golang-build
COPY go.mod go.sum /go/src/gitlab.com/signald/test-infrastructure/
COPY cmd/ /go/src/gitlab.com/signald/test-infrastructure/cmd
WORKDIR /go/src/gitlab.com/signald/test-infrastructure/
RUN go mod download
RUN go build ./cmd/config-generator
FROM debian:buster as build
RUN apt-get update && apt-get install -y openjdk-11-jdk-headless maven git
RUN apt-get update && apt-get install -y openjdk-11-jre-headless
RUN apt-get install -y openjdk-11-jdk-headless maven git
RUN git -C /usr/local/src clone https://github.com/signalapp/Signal-Server
WORKDIR /usr/local/src/Signal-Server
COPY signal-server-patches /tmp/signal-server-patches
@ -8,9 +16,11 @@ RUN mvn install -DskipTests
FROM debian:buster
RUN apt-get update && apt-get install -y openjdk-11-jre-headless
COPY --from=build /usr/local/src/Signal-Server/service/target/TextSecureServer-2.92.jar /usr/share/TextSecureServer.jar
COPY --from=build /usr/local/src/Signal-Server/service/target/TextSecureServer-*.jar /usr/share/TextSecureServer.jar
COPY --from=golang-build /go/src/gitlab.com/signald/test-infrastructure/config-generator /usr/bin/config-generator
COPY migrate-and-start-server.sh /usr/bin/migrate-and-start-server
RUN mkdir -p /etc/signal-server
RUN useradd signal
RUN chown -R signal /usr/share/TextSecureServer.jar
RUN chown -R signal /usr/share/TextSecureServer.jar /etc/signal-server
USER signal
CMD ["/usr/bin/migrate-and-start-server"]

10
test-helper.Dockerfile Normal file
View file

@ -0,0 +1,10 @@
FROM golang:latest as golang-build
COPY go.mod go.sum /go/src/gitlab.com/signald/test-infrastructure/
COPY cmd/ /go/src/gitlab.com/signald/test-infrastructure/cmd
WORKDIR /go/src/gitlab.com/signald/test-infrastructure/
RUN go mod download
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build ./cmd/test-helper
FROM scratch
COPY --from=golang-build /go/src/gitlab.com/signald/test-infrastructure/test-helper /test-helper
ENTRYPOINT ["/test-helper"]

View file

@ -1,8 +0,0 @@
FROM golang:latest as build
COPY testhelper.go /go/testhelper.go
RUN go get -u github.com/lib/pq
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build testhelper.go
FROM scratch
COPY --from=build /go/testhelper /testhelper
ENTRYPOINT ["/testhelper"]