multiple config files support

cmd/sing-box/cmd_format.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/sagernet/sing-box/common/conf"
 	"github.com/sagernet/sing-box/common/json"
 	"github.com/sagernet/sing-box/log"
 	"github.com/sagernet/sing-box/option"
@@ -33,7 +34,16 @@ func init() {
 }
 
 func format() error {
-	configContent, err := os.ReadFile(configPath)
+	files, err := conf.ResolveFiles(configPaths, configRecursive)
+	if err != nil {
+		return E.Cause(err, "resolve config files")
+	}
+	if len(files) == 0 {
+		return E.New("no config file found")
+	}
+	// use conf.Merge even if there's only one config file, make
+	// it has the same behavior between one and multiple files.
+	configContent, err := conf.Merge(files)
 	if err != nil {
 		return E.Cause(err, "read config")
 	}
@@ -49,13 +59,22 @@ func format() error {
 	if err != nil {
 		return E.Cause(err, "encode config")
 	}
+	flagIgnored := false
+	if commandFormatFlagWrite && len(files) > 1 {
+		commandFormatFlagWrite = false
+		flagIgnored = true
+	}
 	if !commandFormatFlagWrite {
 		os.Stdout.WriteString(buffer.String() + "\n")
+		if flagIgnored {
+			log.Warn("--write flag is ignored due to more than one configuration file specified")
+		}
 		return nil
 	}
 	if bytes.Equal(configContent, buffer.Bytes()) {
 		return nil
 	}
+	configPath := files[0]
 	output, err := os.Create(configPath)
 	if err != nil {
 		return E.Cause(err, "open output")

cmd/sing-box/cmd_run.go

@@ -2,13 +2,13 @@ package main
 
 import (
 	"context"
-	"io"
 	"os"
 	"os/signal"
 	runtimeDebug "runtime/debug"
 	"syscall"
 
-	"github.com/sagernet/sing-box"
+	box "github.com/sagernet/sing-box"
+	"github.com/sagernet/sing-box/common/conf"
 	"github.com/sagernet/sing-box/log"
 	"github.com/sagernet/sing-box/option"
 	E "github.com/sagernet/sing/common/exceptions"
@@ -36,10 +36,19 @@ func readConfig() (option.Options, error) {
 		configContent []byte
 		err           error
 	)
-	if configPath == "stdin" {
+	// always use conf.Merge to make it has the same behavior
-		configContent, err = io.ReadAll(os.Stdin)
+	// between one and multiple files.
+	if len(configPaths) == 1 && configPaths[0] == "stdin" {
+		configContent, err = conf.Merge(os.Stdin)
 	} else {
-		configContent, err = os.ReadFile(configPath)
+		files, err := conf.ResolveFiles(configPaths, configRecursive)
+		if err != nil {
+			return option.Options{}, E.Cause(err, "resolve config files")
+		}
+		if len(files) == 0 {
+			return option.Options{}, E.New("no config file found")
+		}
+		configContent, err = conf.Merge(files)
 	}
 	if err != nil {
 		return option.Options{}, E.Cause(err, "read config")

cmd/sing-box/main.go

@@ -10,7 +10,8 @@ import (
 )
 
 var (
-	configPath   string
+	configPaths     []string
+	configRecursive bool
 	workingDir      string
 	disableColor    bool
 )
@@ -21,7 +22,8 @@ var mainCommand = &cobra.Command{
 }
 
 func init() {
-	mainCommand.PersistentFlags().StringVarP(&configPath, "config", "c", "config.json", "set configuration file path")
+	mainCommand.PersistentFlags().StringArrayVarP(&configPaths, "config", "c", []string{"config.json"}, "set configuration files / directories")
+	mainCommand.PersistentFlags().BoolVarP(&configRecursive, "config-recursive", "r", false, "load configuration directories recursively")
 	mainCommand.PersistentFlags().StringVarP(&workingDir, "directory", "D", "", "set working directory")
 	mainCommand.PersistentFlags().BoolVarP(&disableColor, "disable-color", "", false, "disable color output")
 }

common/conf/conf.go

@@ -0,0 +1,133 @@
+package conf
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	myjson "github.com/sagernet/sing-box/common/json"
+	E "github.com/sagernet/sing/common/exceptions"
+
+	"github.com/qjebbs/go-jsons"
+)
+
+var (
+	// formatName is the format name of JSON.
+	formatName jsons.Format = "json"
+	// formatExtensions are the extension names of JSON format.
+	formatExtensions = []string{".json", ".jsonc"}
+)
+
+// Merge merges inputs into a single json.
+func Merge(inputs ...interface{}) ([]byte, error) {
+	return newMerger().MergeAs(formatName, inputs...)
+}
+
+// NewMerger creates a new json files Merger.
+func newMerger() *jsons.Merger {
+	m := jsons.NewMerger()
+	m.RegisterLoader(
+		formatName,
+		formatExtensions,
+		func(b []byte) (map[string]interface{}, error) {
+			m := make(map[string]interface{})
+			decoder := json.NewDecoder(myjson.NewCommentFilter(bytes.NewReader(b)))
+			err := decoder.Decode(&m)
+			if err != nil {
+				return nil, err
+			}
+			return m, nil
+		},
+	)
+	return m
+}
+
+// ResolveFiles expands folder path (if any and it exists) to file paths.
+// Any other paths, like file, even URL, it returns them as is.
+func ResolveFiles(paths []string, recursively bool) ([]string, error) {
+	return resolveFiles(paths, formatExtensions, recursively)
+}
+
+func resolveFiles(paths []string, extensions []string, recursively bool) ([]string, error) {
+	if len(paths) == 0 {
+		return nil, nil
+	}
+	dirReader := readDir
+	if recursively {
+		dirReader = readDirRecursively
+	}
+	files := make([]string, 0)
+	for _, p := range paths {
+		if isRemote(p) {
+			return nil, E.New("remote files are not supported")
+		}
+		if !isDir(p) {
+			files = append(files, p)
+			continue
+		}
+		fs, err := dirReader(p, extensions)
+		if err != nil {
+			return nil, E.Cause(err, "read dir")
+		}
+		files = append(files, fs...)
+	}
+	return files, nil
+}
+
+// readDir finds files according to extensions in the dir
+func readDir(dir string, extensions []string) ([]string, error) {
+	confs, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+	files := make([]string, 0)
+	for _, f := range confs {
+		ext := filepath.Ext(f.Name())
+		for _, e := range extensions {
+			if strings.EqualFold(ext, e) {
+				files = append(files, filepath.Join(dir, f.Name()))
+				break
+			}
+		}
+	}
+	return files, nil
+}
+
+// readDirRecursively finds files according to extensions in the dir recursively
+func readDirRecursively(dir string, extensions []string) ([]string, error) {
+	files := make([]string, 0)
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		ext := filepath.Ext(path)
+		for _, e := range extensions {
+			if strings.EqualFold(ext, e) {
+				files = append(files, path)
+				break
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return files, nil
+}
+
+func isRemote(p string) bool {
+	u, err := url.Parse(p)
+	if err != nil {
+		return false
+	}
+	return u.Scheme == "http" || u.Scheme == "https"
+}
+
+func isDir(p string) bool {
+	i, err := os.Stat(p)
+	if err != nil {
+		return false
+	}
+	return i.IsDir()
+}

common/conf/conf_test.go

@@ -0,0 +1,22 @@
+package conf
+
+import "testing"
+
+func TestIsRemote(t *testing.T) {
+	tests := []struct {
+		path string
+		want bool
+	}{
+		{"http://example.com", true},
+		{"https://example.com/config.json", true},
+		{"config.json", false},
+		{"path/to/config.json", false},
+		{"/path/to/config.json", false},
+		{`d:\config.json`, false},
+	}
+	for _, tt := range tests {
+		if got := isRemote(tt.path); got != tt.want {
+			t.Errorf("isRemote(%s) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}

docs/configuration/index.md

@@ -37,3 +37,70 @@ $ sing-box check
 ```bash
 $ sing-box format -w
 ```
+
+### Multiple Configuration Files
+
+> You can skip this section if you are using only one configuration file.
+
+sing-box supports multiple configuration files. The latter overwrites and merges into the former, in the order in which the configuration files are loaded.
+
+```bash
+# Load by the order of parameters
+sing-box run -c inbound.json -c outbound.json
+# Load by the order of file names
+sing-box run -r -c config_dir
+```
+
+Suppose we have 2 `JSON` files:
+
+`a.json`:
+
+```json
+{
+  "log": {"level": "debug"},
+  "inbounds": [{"tag": "in-1"}],
+  "outbounds": [{"_priority": 100, "tag": "out-1"}],
+  "route": {"rules": [
+    {"_tag":"rule1","inbound":["in-1"],"outbound":"out-1"}
+  ]}
+}
+```
+
+`b.json`:
+
+```json
+{
+  "log": {"level": "error"},
+  "outbounds": [{"_priority": -100, "tag": "out-2"}],
+  "route": {"rules": [
+    {"_tag":"rule1","inbound":["in-1.1"],"outbound":"out-1.1"}
+  ]}
+}
+```
+
+Applied:
+
+```jsonc
+{
+  // level field is overwritten by the latter value
+  "log": {"level": "error"},
+  "inbounds": [{"tag": "in-1"}],
+  "outbounds": [
+    // Although out-2 is a latecomer, but it's in 
+    // the front due to the smaller "_priority"
+    {"tag": "out-2"},
+    {"tag": "out-1"}
+  ],
+  "route": {"rules": [
+    // 2 rules are merged into one due to the same "_tag",
+    // outbound field is overwritten during the merging
+    {"inbound":["in-1","in-1.1"],"outbound":"out-1.1"}
+  ]}
+}
+```
+
+Just remember these few rules:
+
+- Simple values (`string`, `number`, `boolean`) are overwritten, others (`array`, `object`) are merged.
+- Elements with same `_tag` in an array will be merged.
+- Elements in an array will be sorted by `_priority` field, the smaller the higher priority.

docs/configuration/index.zh.md

@@ -37,3 +37,69 @@ $ sing-box check
 ```bash
 $ sing-box format -w
 ```
+
+### 多个配置文件
+
+> 如果只使用单个配置文件，您完全可以忽略这一节。
+
+sing-box 支持多文件配置。按照配置文件的加载顺序，后者会覆盖并合并到前者。
+
+```bash
+# 根据参数顺序加载
+sing-box run -c inbound.json -c outbound.json
+# 根据文件名顺序加载
+sing-box run -r -c config_dir
+```
+
+假设我们有两个 `JSON` 文件：
+
+`a.json`:
+
+```json
+{
+  "log": {"level": "debug"},
+  "inbounds": [{"tag": "in-1"}],
+  "outbounds": [{"_priority": 100, "tag": "out-1"}],
+  "route": {"rules": [
+    {"_tag":"rule1","inbound":["in-1"],"outbound":"out-1"}
+  ]}
+}
+```
+
+`b.json`:
+
+```json
+{
+  "log": {"level": "error"},
+  "outbounds": [{"_priority": -100, "tag": "out-2"}],
+  "route": {"rules": [
+    {"_tag":"rule1","inbound":["in-1.1"],"outbound":"out-1.1"}
+  ]}
+}
+```
+
+合并后:
+
+```jsonc
+{
+  // level 字段被后来者覆盖
+  "log": {"level": "error"},
+  "inbounds": [{"tag": "in-1"}],
+  "outbounds": [
+    // out-2 虽然是后来者，但由于 _priority 小，反而排在前面
+    {"tag": "out-2"}, 
+    {"tag": "out-1"}
+  ],
+  "route": {"rules": [
+    // 2条规则被合并，因为它们具有相同的 "_tag"，
+    // outbound 字段在合并过程被覆盖为 "out-1.1"
+    {"inbound":["in-1","in-1.1"],"outbound":"out-1.1"}
+  ]}
+}
+```
+
+只需记住这几个规则：
+
+- 简单字段（字符串、数字、布尔值）会被后来者覆盖，其它字段（数组、对象）会被合并。
+- 数组内拥有相同 `_tag` 的对象会被合并。
+- 数组会按 `_priority` 字段值进行排序，越小的优先级越高。

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/miekg/dns v1.1.50
 	github.com/oschwald/maxminddb-golang v1.10.0
 	github.com/pires/go-proxyproto v0.6.2
+	github.com/qjebbs/go-jsons v0.0.0-20221021030617-ee43abc0a749
 	github.com/refraction-networking/utls v1.1.5
 	github.com/sagernet/quic-go v0.0.0-20220818150011-de611ab3e2bb
 	github.com/sagernet/sing v0.0.0-20221008120626-60a9910eefe4

go.sum

@@ -131,6 +131,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/qjebbs/go-jsons v0.0.0-20221021030617-ee43abc0a749 h1:WVxhEUqfIi/euX8eVRfl7CBtewZkv88DiIulPjzJfrU=
+github.com/qjebbs/go-jsons v0.0.0-20221021030617-ee43abc0a749/go.mod h1:wNJrtinHyC3YSf6giEh4FJN8+yZV7nXBjvmfjhBIcw4=
 github.com/refraction-networking/utls v1.1.5 h1:JtrojoNhbUQkBqEg05sP3gDgDj6hIEAAVKbI9lx4n6w=
 github.com/refraction-networking/utls v1.1.5/go.mod h1:jRQxtYi7nkq1p28HF2lwOH5zQm9aC8rpK0O9lIIzGh8=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=