diff --git a/tools/fxconfig/.gitignore b/tools/fxconfig/.gitignore index 5d56d838..f05e6df3 100644 --- a/tools/fxconfig/.gitignore +++ b/tools/fxconfig/.gitignore @@ -1 +1,2 @@ /integration/testdata/crypto/ +/integration/testdata/crypto_backup/ diff --git a/tools/fxconfig/integration/helpers_test.go b/tools/fxconfig/integration/helpers_test.go index c2e9d492..ab4c0bc5 100644 --- a/tools/fxconfig/integration/helpers_test.go +++ b/tools/fxconfig/integration/helpers_test.go @@ -10,9 +10,13 @@ import ( "bytes" "fmt" "os" + "os/exec" "path/filepath" + "runtime" + "slices" "strings" "testing" + "time" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" @@ -40,20 +44,28 @@ const ( // and returns a map containing the endpoints of the committers services. func setupSingleOrgAdmin(t *testing.T) map[string]string { t.Helper() + ensureIntegrationTestdata(t) - genesisPath, err := filepath.Abs(filepath.Join(".", "testdata", "crypto", "single-org.pb.bin")) - require.NoError(t, err) + _, filename, _, ok := runtime.Caller(0) + require.True(t, ok) + + baseDir := filepath.Dir(filename) + genesisPath := filepath.Join(baseDir, "testdata", "crypto", "single-org.pb.bin") return setup(t, genesisPath) } -// setupSingleOrgAdmin spawns a committer test container with a single org admin lifecycle policy +// setupMultiOrgAdmin spawns a committer test container with a multi-org admin lifecycle policy // and returns a map containing the endpoints of the committers services. func setupMultiOrgAdmin(t *testing.T) map[string]string { t.Helper() + ensureIntegrationTestdata(t) - genesisPath, err := filepath.Abs(filepath.Join(".", "testdata", "crypto", "multi-org.pb.bin")) - require.NoError(t, err) + _, filename, _, ok := runtime.Caller(0) + require.True(t, ok) + + baseDir := filepath.Dir(filename) + genesisPath := filepath.Join(baseDir, "testdata", "crypto", "multi-org.pb.bin") return setup(t, genesisPath) } @@ -61,51 +73,229 @@ func setupMultiOrgAdmin(t *testing.T) map[string]string { func setup(t *testing.T, genesisPath string) map[string]string { t.Helper() - dataDirectory, err := filepath.Abs(filepath.Join(".", "testdata", "crypto")) - require.NoError(t, err) + _, filename, _, ok := runtime.Caller(0) + require.True(t, ok) + + baseDir := filepath.Dir(filename) + dataDirectory := filepath.Join(baseDir, "testdata", "crypto") + hostOrdererDir := resolveHostOrdererDir(dataDirectory) + require.NotEmpty(t, hostOrdererDir, "missing orderer directory under %s", dataDirectory) + + // Orderer TLS cert/key/CA (test host paths). + // Generated by cryptogen for orderer.orderer.com, mounted at /server-certs/ + // which is the path the container's default mock-orderer.yaml expects. + ordererTLSDir := filepath.Join(hostOrdererDir, "tls") + ordererCertPath := filepath.Join(ordererTLSDir, "server.crt") + ordererKeyPath := filepath.Join(ordererTLSDir, "server.key") + ordererCAPath := filepath.Join(ordererTLSDir, "ca.crt") + clientOrdererCAPath := "/client-certs/ca-certificate.pem" + + ordererMSPBasePath := "/root/artifacts/ordererOrganizations/OrdererOrg/orderers/consenter-rg/msp" + ordererOrgMSPBasePath := "/root/artifacts/ordererOrganizations/OrdererOrg/msp" + ordererSigncertPath := filepath.Join(ordererMSPBasePath, "signcerts", "orderer.orderer.com-cert.pem") + ordererCACertPath := filepath.Join(ordererMSPBasePath, "cacerts", "ca.orderer.com-cert.pem") + ordererTLSCACertPath := filepath.Join(ordererMSPBasePath, "tlscacerts", "tlsca.orderer.com-cert.pem") + ordererKeystorePath := filepath.Join(ordererMSPBasePath, "keystore", "priv_sk") + ordererAdminCertPath := filepath.Join(ordererMSPBasePath, "admincerts", "Admin@orderer.com-cert.pem") + + hostOrdererMSPDir := filepath.Join(hostOrdererDir, "msp") + + hostOrdererSigncertPath := findExistingFile( + filepath.Join(hostOrdererMSPDir, "signcerts", "orderer.orderer.com-cert.pem"), + filepath.Join(hostOrdererMSPDir, "signcerts", "consenter-rg-cert.pem"), + ) + require.NotEmpty(t, hostOrdererSigncertPath, "missing orderer signcert under %s", hostOrdererMSPDir) + + hostOrdererCACertPath := findExistingFile( + filepath.Join(hostOrdererMSPDir, "cacerts", "ca.orderer.com-cert.pem"), + ) + require.NotEmpty(t, hostOrdererCACertPath, "missing orderer CA cert under %s", hostOrdererMSPDir) + + hostOrdererTLSCACertPath := findExistingFile( + filepath.Join(hostOrdererMSPDir, "tlscacerts", "tlsca.orderer.com-cert.pem"), + ) + require.NotEmpty(t, hostOrdererTLSCACertPath, "missing orderer TLS CA cert under %s", hostOrdererMSPDir) + + hostOrdererKeystorePath := findExistingFile( + filepath.Join(hostOrdererMSPDir, "keystore", "priv_sk"), + ) + require.NotEmpty(t, hostOrdererKeystorePath, "missing orderer keystore key under %s", hostOrdererMSPDir) + + hostOrdererAdminCertPath := findExistingFile( + filepath.Join(hostOrdererMSPDir, "admincerts", "Admin@orderer.com-cert.pem"), + ) + require.NotEmpty(t, hostOrdererAdminCertPath, "missing orderer admin cert under %s", hostOrdererMSPDir) + + // The orderer binary reads its signing MSP from: + // artifacts-path/ordererOrganizations/OrdererOrg/orderers/consenter-rg/msp + // Our testdata has consenter-rg/msp/keystore but is missing cacerts and signcerts. + // Individual cert files from orderer.orderer.com/msp are mounted below to fill the gap. - // msp configuration for sidecar orderer client + // Sidecar identity inside the container. + // testcontainers WithFiles for a directory copies the directory AS A SUBDIRECTORY + // of the destination. So cryptoDir copied to "/root/artifacts/" lands at + // "/root/artifacts/crypto/" (the original code confirmed this path). mspID := "Org1MSP" mspDir := "/root/artifacts/crypto/peerOrganizations/Org1/users/committer@org1.com/msp" ctx := t.Context() committerContainer, err := testcontainers.Run( ctx, "ghcr.io/hyperledger/fabric-x-committer-test-node:0.1.9", - testcontainers.WithCmd("run", "db", "orderer", "committer", "--insecure"), + // Run WITHOUT --insecure so that TLS is active for the orderer. + // All internal service-to-service connections are explicitly set to "none" + // below, so only the orderer server TLS path is exercised. + testcontainers.WithCmd("run", "db", "orderer", "committer"), + // Mount only the genesis block — do NOT mount all of testdata/crypto at + // /root/artifacts/ because that would overwrite the container's built-in + // orderer MSP artifacts (needed by the orderer binary to sign blocks). testcontainers.WithFiles(testcontainers.ContainerFile{ HostFilePath: genesisPath, ContainerFilePath: "/root/artifacts/config-block.pb.bin", FileMode: 0o700, }), + // Mount testdata/crypto at /root/artifacts/ so that peerOrganizations/ + // and ordererOrganizations/ are accessible from the default artifacts path. testcontainers.WithFiles(testcontainers.ContainerFile{ HostFilePath: dataDirectory, ContainerFilePath: "/root/artifacts/", FileMode: 0o755, }), + // The orderer binary resolves its signing MSP via the built-in container path: + // /root/artifacts/ordererOrganizations/OrdererOrg/orderers/consenter-rg/msp + // (This is the IMAGE's path, not under /root/artifacts/crypto/, because it's + // part of the built-in artifact tree in the container image.) + // Our testdata's consenter-rg/msp only has a keystore (no cacerts or signcerts). + // We inject the needed cert files from orderer.orderer.com/msp individually. + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: hostOrdererSigncertPath, + ContainerFilePath: ordererSigncertPath, + FileMode: 0o644, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: hostOrdererCACertPath, + ContainerFilePath: ordererCACertPath, + FileMode: 0o644, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: hostOrdererCACertPath, + ContainerFilePath: filepath.Join(ordererOrgMSPBasePath, "cacerts", "ca.orderer.com-cert.pem"), + FileMode: 0o644, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: hostOrdererTLSCACertPath, + ContainerFilePath: ordererTLSCACertPath, + FileMode: 0o644, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: hostOrdererTLSCACertPath, + ContainerFilePath: filepath.Join(ordererOrgMSPBasePath, "tlscacerts", "tlsca.orderer.com-cert.pem"), + FileMode: 0o644, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: hostOrdererKeystorePath, + ContainerFilePath: ordererKeystorePath, + FileMode: 0o600, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: hostOrdererAdminCertPath, + ContainerFilePath: ordererAdminCertPath, + FileMode: 0o644, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: hostOrdererAdminCertPath, + ContainerFilePath: filepath.Join(ordererOrgMSPBasePath, "admincerts", "Admin@orderer.com-cert.pem"), + FileMode: 0o644, + }), + // Mount orderer TLS certs at the paths the container's mock-orderer.yaml expects: + // /server-certs/public-key.pem (server TLS cert) + // /server-certs/private-key.pem (server TLS key) + // /server-certs/ca-certificate.pem (TLS CA cert, used by sidecar to verify orderer) + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: ordererCertPath, + ContainerFilePath: "/server-certs/public-key.pem", + FileMode: 0o600, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: ordererKeyPath, + ContainerFilePath: "/server-certs/private-key.pem", + FileMode: 0o600, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: ordererCAPath, + ContainerFilePath: "/server-certs/ca-certificate.pem", + FileMode: 0o600, + }), + testcontainers.WithFiles(testcontainers.ContainerFile{ + HostFilePath: ordererCAPath, + ContainerFilePath: clientOrdererCAPath, + FileMode: 0o600, + }), testcontainers.WithExposedPorts(ordererPort, sidecarPort, queryServicePort), testcontainers.WithEnv(map[string]string{ - "SC_COORDINATOR_LOGGING_LOGSPEC": "DEBUG", - "SC_SIDECAR_LOGGING_LOGSPEC": "DEBUG", + "SC_COORDINATOR_LOGGING_LOGSPEC": "DEBUG", + "SC_SIDECAR_LOGGING_LOGSPEC": "DEBUG", + + // Sidecar identity for signing envelopes submitted to the orderer. "SC_SIDECAR_ORDERER_CHANNEL_ID": channelID, - "SC_SIDECAR_ORDERER_TLS_MODE": "none", "SC_SIDECAR_ORDERER_SIGNED_ENVELOPES": "true", "SC_SIDECAR_ORDERER_IDENTITY_MSP_ID": mspID, "SC_SIDECAR_ORDERER_IDENTITY_MSP_DIR": mspDir, - "SC_QUERY_SERVICE_SERVER_ENDPOINT": fmt.Sprintf(":%v", queryServicePort), - "SC_QUERY_SERVICE_LOGGING_LOGSPEC": "DEBUG", - "SC_ORDERER_BLOCK_SIZE": "1", - "SC_ORDERER_LOGGING_LOGSPEC": "DEBUG", - "SC_VC_LOGGING_LOGSPEC": "DEBUG", + + // ── Orderer Server TLS ────────────────────────────────────────────── + // Use one-way TLS ("tls") so clients only need the server CA cert + // and do not have to present a client certificate. The server cert/key + // are already mounted at the paths mock-orderer.yaml expects. + "SC_ORDERER_SERVER_TLS_MODE": "tls", + "SC_ORDERER_LOGGING_LOGSPEC": "DEBUG", + "SC_ORDERER_BLOCK_SIZE": "1", + + // ── Sidecar → Orderer TLS ─────────────────────────────────────────── + // The sidecar connects to the orderer as a gRPC client. + // "tls" mode = server-auth only (no client cert required from sidecar). + "SC_SIDECAR_ORDERER_TLS_MODE": "tls", + // The sidecar always loads cert/key files even in one-way TLS mode. + // Point them to the already-mounted orderer cert so the open() succeeds. + // These values are not used for authentication since mode is "tls". + "SC_SIDECAR_ORDERER_TLS_CERT_PATH": "/server-certs/public-key.pem", + "SC_SIDECAR_ORDERER_TLS_KEY_PATH": "/server-certs/private-key.pem", + + // ── Internal Service TLS (disabled) ──────────────────────────────── + // Connections between coordinator, vc, verifier, query, and sidecar are + // internal to the container and do not need TLS for these tests. + // Disabling them avoids the need to provision /client-certs/ with certs. + "SC_COORDINATOR_VERIFIER_TLS_MODE": "none", + "SC_COORDINATOR_VALIDATOR_COMMITTER_TLS_MODE": "none", + "SC_COORDINATOR_SERVER_TLS_MODE": "none", + "SC_COORDINATOR_MONITORING_TLS_MODE": "none", + "SC_SIDECAR_SERVER_TLS_MODE": "none", + "SC_SIDECAR_MONITORING_TLS_MODE": "none", + "SC_SIDECAR_COMMITTER_TLS_MODE": "none", + "SC_QUERY_SERVER_TLS_MODE": "none", + "SC_QUERY_MONITORING_TLS_MODE": "none", + "SC_VC_SERVER_TLS_MODE": "none", + "SC_VC_MONITORING_TLS_MODE": "none", + "SC_VERIFIER_SERVER_TLS_MODE": "none", + "SC_VERIFIER_MONITORING_TLS_MODE": "none", + + "SC_QUERY_SERVICE_SERVER_ENDPOINT": fmt.Sprintf(":%v", queryServicePort), + "SC_QUERY_SERVICE_LOGGING_LOGSPEC": "DEBUG", + "SC_VC_LOGGING_LOGSPEC": "DEBUG", }), - testcontainers.WithWaitStrategy( - wait.ForListeningPort(ordererPort), - wait.ForListeningPort(sidecarPort), - wait.ForListeningPort(queryServicePort), + testcontainers.WithWaitStrategyAndDeadline( + 3*time.Minute, + // Orderer server is ready when it logs its gRPC server start. + wait.ForLog("Listening on: tcp://:7050"), + // Sidecar gRPC server is ready. + wait.ForLog("Listening on: tcp://:4001"), + // Query service is ready. + wait.ForLog("Listening on: tcp://:7001"), + // Sidecar has fetched the genesis block and synced with the coordinator. + // This is the signal that TLS is fully working end-to-end. wait.ForLog("Setting the last committed block number:"), ), ) t.Cleanup(func() { - testcontainers.CleanupContainer(t, committerContainer) + // testcontainers.CleanupContainer(t, committerContainer) }) require.NoError(t, err) @@ -119,9 +309,219 @@ func setup(t *testing.T, genesisPath string) map[string]string { endpoints["sidecar"], err = committerContainer.PortEndpoint(ctx, sidecarPort, "") require.NoError(t, err) + // Resolve the absolute path to the orderer CA cert on the test host. + // The fxconfig client uses it to validate the orderer's TLS server certificate. + ordererCAAbsPath, err := filepath.Abs(ordererCAPath) + require.NoError(t, err) + endpoints["ordererCA"] = ordererCAAbsPath + return endpoints } +func ensureIntegrationTestdata(t *testing.T) { + t.Helper() + + _, filename, _, ok := runtime.Caller(0) + require.True(t, ok) + + baseDir := filepath.Dir(filename) + + for range 2 { + if integrationTestdataReady(baseDir) { + return + } + + t.Logf( + "integration testdata not ready yet under %s; regenerating", + filepath.Join(baseDir, "testdata", "crypto"), + ) + generateIntegrationTestdata(t, baseDir) + } + + cryptoDir := filepath.Join(baseDir, "testdata", "crypto") + + require.Truef( + t, + integrationTestdataReady(baseDir), + "integration testdata preflight incomplete under %s (missing: %s)", + cryptoDir, + strings.Join(integrationTestdataMissing(baseDir), ", "), + ) +} + +func generateIntegrationTestdata(t *testing.T, baseDir string) { + t.Helper() + + repoRoot := resolveRepoRoot(baseDir) + require.NotEmpty(t, repoRoot, "failed to locate repository root from %s", baseDir) + cryptoConfigPath := "tools/fxconfig/integration/testdata/crypto-config.yaml" + cryptoOutputPath := "tools/fxconfig/integration/testdata/crypto" + testdataPath := "tools/fxconfig/integration/testdata" + singleOrgBlockPath := "tools/fxconfig/integration/testdata/crypto/single-org.pb.bin" + multiOrgBlockPath := "tools/fxconfig/integration/testdata/crypto/multi-org.pb.bin" + + // Ensure cryptogen/configtxgen always work from a clean output tree. + // Stale files can leave MSP cert chains inconsistent and produce invalid genesis blocks. + require.NoError(t, os.RemoveAll(filepath.Join(repoRoot, cryptoOutputPath))) + + commands := [][]string{ + {"go", "run", "./tools/cryptogen", "generate", "--config", cryptoConfigPath, "--output", cryptoOutputPath}, + { + "go", "run", "./tools/configtxgen", + "-configPath", testdataPath, + "-channelID", "mychannel", + "-profile", "SingleOrgAdminChannel", + "-outputBlock", singleOrgBlockPath, + }, + { + "go", "run", "./tools/configtxgen", + "-configPath", testdataPath, + "-channelID", "mychannel", + "-profile", "MultiOrgAdminChannel", + "-outputBlock", multiOrgBlockPath, + }, + } + + for _, args := range commands { + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = repoRoot + output, err := cmd.CombinedOutput() + require.NoErrorf(t, err, "failed to generate integration testdata with %v:\n%s", args, string(output)) + } +} + +func resolveRepoRoot(baseDir string) string { + dir, err := filepath.Abs(baseDir) + if err != nil { + return "" + } + + for { + if fileExists(filepath.Join(dir, "go.mod")) { + return dir + } + + parent := filepath.Dir(dir) + if parent == dir { + return "" + } + dir = parent + } +} + +func integrationTestdataReady(baseDir string) bool { + cryptoDir := filepath.Join(baseDir, "testdata", "crypto") + if !fileExists(filepath.Join(cryptoDir, "single-org.pb.bin")) { + return false + } + if !fileExists(filepath.Join(cryptoDir, "multi-org.pb.bin")) { + return false + } + + return resolveHostOrderersRoot(cryptoDir) != "" +} + +func integrationTestdataMissing(baseDir string) []string { + cryptoDir := filepath.Join(baseDir, "testdata", "crypto") + missing := make([]string, 0) + + if !fileExists(filepath.Join(cryptoDir, "single-org.pb.bin")) { + missing = append(missing, filepath.Join(cryptoDir, "single-org.pb.bin")) + } + if !fileExists(filepath.Join(cryptoDir, "multi-org.pb.bin")) { + missing = append(missing, filepath.Join(cryptoDir, "multi-org.pb.bin")) + } + if resolveHostOrderersRoot(cryptoDir) == "" { + missing = append(missing, "ordererOrganizations/*/orderers") + } + + if len(missing) == 0 { + missing = append(missing, "") + } + + return missing +} + +func resolveHostOrdererDir(dataDirectory string) string { + orderersRoot := resolveHostOrderersRoot(dataDirectory) + if orderersRoot == "" { + return "" + } + + entries, err := os.ReadDir(orderersRoot) + if err != nil { + return "" + } + + var candidates []string + for _, entry := range entries { + if entry.IsDir() { + candidates = append(candidates, filepath.Join(orderersRoot, entry.Name())) + } + } + slices.Sort(candidates) + + for _, candidate := range candidates { + if dirExists(filepath.Join(candidate, "msp")) && dirExists(filepath.Join(candidate, "tls")) { + return candidate + } + } + + for _, candidate := range candidates { + if dirExists(filepath.Join(candidate, "msp")) { + return candidate + } + } + + return "" +} + +func resolveHostOrderersRoot(dataDirectory string) string { + ordererOrgsRoot := filepath.Join(dataDirectory, "ordererOrganizations") + entries, err := os.ReadDir(ordererOrgsRoot) + if err != nil { + return "" + } + + var orgDirs []string + for _, entry := range entries { + if entry.IsDir() { + orgDirs = append(orgDirs, entry.Name()) + } + } + slices.Sort(orgDirs) + + for _, orgDir := range orgDirs { + orderersRoot := filepath.Join(ordererOrgsRoot, orgDir, "orderers") + if dirExists(orderersRoot) { + return orderersRoot + } + } + + return "" +} + +func fileExists(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() +} + +func dirExists(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +func findExistingFile(paths ...string) string { + for _, path := range paths { + info, err := os.Stat(path) + if err == nil && !info.IsDir() { + return path + } + } + + return "" +} + func generateConfigFile( tb testing.TB, localMspID string, @@ -131,6 +531,9 @@ func generateConfigFile( tb.Helper() tmpDir := tb.TempDir() configPath := filepath.Join(tmpDir, "config.yaml") + + // Enable TLS only for the orderer connection; query and notifications run + // without TLS (SC_QUERY_SERVER_TLS_MODE=none, SC_SIDECAR_SERVER_TLS_MODE=none). configContent := ` msp: localMspID: ` + localMspID + ` @@ -140,6 +543,10 @@ orderer: address: ` + endpoints["orderer"] + ` channel: ` + channelID + ` connectionTimeout: 30s + tls: + enabled: true + rootCerts: + - ` + endpoints["ordererCA"] + ` queries: address: ` + endpoints["query"] + `