log(dm): close sensitive diagnostic redaction gaps

cmd/dm-worker/main.go

@@ -47,12 +47,12 @@ func main() {
 		os.Exit(2)
 	}
 
+	log.SetRedactLog(cfg.RedactInfoLog)
 	err = log.InitLogger(&log.Config{
 		File:   cfg.LogFile,
 		Format: cfg.LogFormat,
 		Level:  strings.ToLower(cfg.LogLevel),
 	})
-	log.SetRedactLog(cfg.RedactInfoLog)
 	if err != nil {
 		common.PrintLinesf("init logger error %s", terror.Message(err))
 		os.Exit(2)

dm/checker/checker.go

@@ -819,7 +819,7 @@ func (c *Checker) IsFreshTask() (bool, error) {
 	}
 	var existCheckpoint bool
 	for _, sql := range checkpointSQLs {
-		c.tctx.Logger.Info("exec query", zap.String("sql", sql))
+		c.tctx.Logger.Info("exec query", log.RedactField(zap.String("sql", sql)))
 		rows, err := instance.targetDB.DB.QueryContext(c.tctx.Ctx, sql)
 		if err != nil {
 			if conn.IsMySQLError(err, mysql.ErrNoSuchTable) {

dm/config/security/security.go

@@ -80,11 +80,15 @@ func (s *Security) LoadTLSContent() error {
 	return firstErr
 }
 
-// ClearSSLBytesData clear all tls config bytes data.
+// ClearSSLBytesData clears TLS contents in both byte and Base64 representations
+// before displaying configuration. Certificate paths remain available.
 func (s *Security) ClearSSLBytesData() {
 	s.SSLCABytes = s.SSLCABytes[:0]
 	s.SSLKeyBytes = s.SSLKeyBytes[:0]
 	s.SSLCertBytes = s.SSLCertBytes[:0]
+	s.SSLCABase64 = ""
+	s.SSLKeyBase64 = ""
+	s.SSLCertBase64 = ""
 }
 
 // Clone returns a deep copy of Security.

dm/config/security_test.go

@@ -106,6 +106,14 @@ func (c *testTLSConfig) TestLoadAndClearContent() {
 	c.Require().Len(s.SSLCABytes, 0)
 	c.Require().Len(s.SSLCertBytes, 0)
 	c.Require().Len(s.SSLKeyBytes, 0)
+	s.SSLCABase64, s.SSLCertBase64, s.SSLKeyBase64 = "private-ca", "private-cert", "private-key"
+	clone := s.Clone()
+	clone.ClearSSLBytesData()
+	c.Require().Empty(clone.SSLCABase64)
+	c.Require().Empty(clone.SSLCertBase64)
+	c.Require().Empty(clone.SSLKeyBase64)
+	c.Require().Equal("private-key", s.SSLKeyBase64)
+	s.ClearSSLBytesData()
 
 	s.SSLCABase64 = "MTIz"
 	err = s.LoadTLSContent()

dm/config/source_config.go

@@ -56,7 +56,7 @@ var SampleSourceConfig string
 
 // ObfuscatedPasswordForFeedback is the source encryption password that returns to the foreground.
 // PM's requirement, we always return obfuscated password to users.
-var ObfuscatedPasswordForFeedback string = "******"
+var ObfuscatedPasswordForFeedback string = log.RedactedCredential
 
 // PurgeConfig is the configuration for Purger.
 type PurgeConfig struct {

dm/dumpling/dumpling.go

@@ -60,7 +60,7 @@ type Dumpling struct {
 func NewDumpling(cfg *config.SubTaskConfig) *Dumpling {
 	logger := log.L()
 	if cfg.FrameworkLogger != nil {
-		logger = log.Logger{Logger: cfg.FrameworkLogger}
+		logger = log.Logger{Logger: log.WithRedaction(cfg.FrameworkLogger)}
 	}
 	m := &Dumpling{
 		cfg:    cfg,
@@ -108,7 +108,7 @@ func (m *Dumpling) Init(ctx context.Context) error {
 		m.dumpConfig.PromRegistry = prometheus.DefaultGatherer.(prometheus.Registerer)
 	}
 
-	m.logger.Info("create dumpling", zap.Stringer("config", m.dumpConfig))
+	m.logger.Info("create dumpling", zap.String("config", log.RedactConfigJSON(m.dumpConfig.String())))
 	return nil
 }
 

dm/loader/lightning.go

@@ -90,7 +90,7 @@ func NewLightning(cfg *config.SubTaskConfig, cli *clientv3.Client, workerName st
 	lightningCfg := MakeGlobalConfig(cfg)
 	logger := log.L()
 	if cfg.FrameworkLogger != nil {
-		logger = log.Logger{Logger: cfg.FrameworkLogger}
+		logger = log.Logger{Logger: log.WithRedaction(cfg.FrameworkLogger)}
 	}
 	loader := &LightningLoader{
 		cfg:                   cfg,
@@ -306,7 +306,7 @@ func (l *LightningLoader) runLightning(ctx context.Context, cfg *lcfg.Config) (e
 			lserver.WithDumpFileStorage(l.cfg.ExtStorage))
 	}
 	if l.cfg.FrameworkLogger != nil {
-		opts = append(opts, lserver.WithLogger(l.cfg.FrameworkLogger))
+		opts = append(opts, lserver.WithLogger(log.WithRedaction(l.cfg.FrameworkLogger)))
 	} else {
 		opts = append(opts, lserver.WithLogger(l.logger.Logger))
 	}

dm/master/server.go

@@ -2524,6 +2524,7 @@ func (s *Server) GetCfg(ctx context.Context, req *pb.GetCfgRequest) (*pb.GetCfgR
 		// For the get-config command, we want to filter out fields that are not easily readable by humans,
 		// such as SSLXXBytes field in `Security` struct
 		taskCfg := config.SubTaskConfigsToTaskConfig(subCfgList...)
+		taskCfg.TargetDB = taskCfg.TargetDB.Clone()
 		taskCfg.TargetDB.Password = config.ObfuscatedPasswordForFeedback
 		if taskCfg.TargetDB.Security != nil {
 			taskCfg.TargetDB.Security.ClearSSLBytesData()
@@ -2628,6 +2629,7 @@ func (s *Server) GetCfg(ctx context.Context, req *pb.GetCfgRequest) (*pb.GetCfgR
 
 			return resp2, nil
 		}
+		sourceCfg.From = *sourceCfg.From.Clone()
 		sourceCfg.From.Password = config.ObfuscatedPasswordForFeedback
 		if sourceCfg.From.Security != nil {
 			sourceCfg.From.Security.ClearSSLBytesData()

dm/pb/hide_password.go

@@ -14,18 +14,163 @@
 package pb
 
 import (
-	"github.com/pingcap/tiflow/pkg/logutil"
+	"bytes"
+	"io"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/pingcap/tiflow/dm/pkg/log"
 	"go.uber.org/zap/zapcore"
+	"gopkg.in/yaml.v2"
 )
 
 // MarshalLogObject implements zapcore.ObjectMarshaler.
 func (m *StartTaskRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
-	enc.AddString("HidePasswordObject", logutil.HideSensitive(m.String()))
+	clone := *m
+	clone.Task = redactYAMLConfig(m.Task)
+	enc.AddString("HidePasswordObject", clone.String())
+	return nil
+}
+
+// MarshalLogObject implements zapcore.ObjectMarshaler.
+func (m *CheckTaskRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
+	clone := *m
+	clone.Task = redactYAMLConfig(m.Task)
+	enc.AddString("HidePasswordObject", clone.String())
 	return nil
 }
 
 // MarshalLogObject implements zapcore.ObjectMarshaler.
 func (m *OperateSourceRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
-	enc.AddString("HidePasswordObject", logutil.HideSensitive(m.String()))
+	clone := *m
+	clone.Config = make([]string, len(m.Config))
+	for i, cfg := range m.Config {
+		clone.Config[i] = redactYAMLConfig(cfg)
+	}
+	enc.AddString("HidePasswordObject", clone.String())
+	return nil
+}
+
+// MarshalLogObject implements zapcore.ObjectMarshaler.
+func (m *CheckSubtasksCanUpdateRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
+	clone := *m
+	clone.SubtaskCfgTomlString = redactTOMLConfig(m.SubtaskCfgTomlString)
+	enc.AddString("HidePasswordObject", clone.String())
+	return nil
+}
+
+// MarshalLogObject implements zapcore.ObjectMarshaler.
+func (m *HandleWorkerErrorRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
+	clone := *m
+	clone.Sqls = redactSQLs(m.Sqls)
+	enc.AddString("payload", clone.String())
+	return nil
+}
+
+// MarshalLogObject implements zapcore.ObjectMarshaler.
+func (m *OperateWorkerSchemaRequest) MarshalLogObject(enc zapcore.ObjectEncoder) error {
+	clone := *m
+	if clone.Schema != "" {
+		clone.Schema = log.RedactString(clone.Schema)
+	}
+	enc.AddString("payload", clone.String())
 	return nil
 }
+
+func redactSQLs(sqls []string) []string {
+	if !log.IsRedactLogEnabled() {
+		return sqls
+	}
+	clone := make([]string, len(sqls))
+	for i := range clone {
+		clone[i] = "?"
+	}
+	return clone
+}
+
+const invalidConfigForLog = "(invalid config; redacted)"
+
+// Decode generic mappings instead of config structs so diagnostics retain unknown
+// fields without applying defaults. Never include decoder errors: they can quote
+// sensitive input. Only the log representation is changed.
+func redactYAMLConfig(input string) string {
+	if input == "" {
+		return input
+	}
+	var cfg map[string]any
+	decoder := yaml.NewDecoder(strings.NewReader(input))
+	decoder.SetStrict(true)
+	if err := decoder.Decode(&cfg); err != nil {
+		return invalidConfigForLog
+	}
+	// Task and source configs are single documents. Reject trailing documents
+	// rather than accidentally logging only part of a request.
+	var extra any
+	if err := decoder.Decode(&extra); err != io.EOF {
+		return invalidConfigForLog
+	}
+	redactConfigFields(cfg)
+	output, err := yaml.Marshal(cfg)
+	if err != nil {
+		return invalidConfigForLog
+	}
+	return string(output)
+}
+
+func redactTOMLConfig(input string) string {
+	if input == "" {
+		return input
+	}
+	var cfg map[string]any
+	if _, err := toml.Decode(input, &cfg); err != nil {
+		return invalidConfigForLog
+	}
+	redactConfigFields(cfg)
+	var output bytes.Buffer
+	if err := toml.NewEncoder(&output).Encode(cfg); err != nil {
+		return invalidConfigForLog
+	}
+	return output.String()
+}
+
+func isSensitiveConfigField(key string) bool {
+	if log.IsRedactLogEnabled() && log.IsSensitiveExpressionField(key) {
+		return true
+	}
+	switch strings.ToLower(key) {
+	case "password", "ssl-ca-bytes", "ssl-cert-bytes", "ssl-key-bytes",
+		"ssl-ca-base64", "ssl-cert-base64", "ssl-key-base64":
+		return true
+	default:
+		return false
+	}
+}
+
+func redactConfigFields(value any) {
+	switch value := value.(type) {
+	case map[string]any:
+		for key, child := range value {
+			if isSensitiveConfigField(key) {
+				value[key] = log.RedactedCredential
+			} else {
+				redactConfigFields(child)
+			}
+		}
+	case map[any]any: // Nested YAML mappings.
+		for key, child := range value {
+			if name, ok := key.(string); ok && isSensitiveConfigField(name) {
+				value[key] = log.RedactedCredential
+			} else {
+				redactConfigFields(child)
+			}
+		}
+	case []any:
+		for _, child := range value {
+			redactConfigFields(child)
+		}
+	case []map[string]any: // TOML arrays of tables.
+		for _, child := range value {
+			redactConfigFields(child)
+		}
+	}
+}