From d95a2dd87b174597b3ccd82f6043110f09fdedc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Wed, 2 Sep 2026 22:07:24 +0800 Subject: [PATCH] Load rule-sets through mmap on iOS --- common/ipset/set.go | 149 +++++++ common/srs/binary.go | 126 ++++-- common/srs/compat_test.go | 16 +- common/srs/ip_set.go | 101 +++-- common/srs/mmap.go | 403 ++++++++++++++++++ common/srs/mmap_stub.go | 13 + common/srs/mmap_test.go | 80 ++++ common/srs/mmap_unix.go | 20 + option/rule_set.go | 7 +- route/rule/rule_item_cidr.go | 7 +- route/rule/rule_set.go | 2 +- route/rule/rule_set_local.go | 3 +- route/rule/rule_set_mmap_ios.go | 44 ++ route/rule/rule_set_mmap_stub.go | 14 + route/rule/rule_set_remote.go | 2 +- route/rule/rule_set_update_validation_test.go | 9 +- 16 files changed, 893 insertions(+), 103 deletions(-) create mode 100644 common/ipset/set.go create mode 100644 common/srs/mmap.go create mode 100644 common/srs/mmap_stub.go create mode 100644 common/srs/mmap_test.go create mode 100644 common/srs/mmap_unix.go create mode 100644 route/rule/rule_set_mmap_ios.go create mode 100644 route/rule/rule_set_mmap_stub.go diff --git a/common/ipset/set.go b/common/ipset/set.go new file mode 100644 index 000000000..ed5e5a438 --- /dev/null +++ b/common/ipset/set.go @@ -0,0 +1,149 @@ +package ipset + +import ( + "encoding/binary" + "net/netip" + "slices" + + E "github.com/sagernet/sing/common/exceptions" + + "go4.org/netipx" +) + +type Range4 struct { + From uint32 + To uint32 +} + +type Range6 struct { + From [16]byte + To [16]byte +} + +type Set struct { + ranges4 []Range4 + ranges6 []Range6 + storage any +} + +func FromIPSet(set *netipx.IPSet) *Set { + result := &Set{} + for _, ipRange := range set.Ranges() { + from := ipRange.From() + to := ipRange.To() + if from.Is4() { + result.ranges4 = append(result.ranges4, Range4{ + From: binary.BigEndian.Uint32(from.AsSlice()), + To: binary.BigEndian.Uint32(to.AsSlice()), + }) + } else { + result.ranges6 = append(result.ranges6, Range6{ + From: from.As16(), + To: to.As16(), + }) + } + } + return result +} + +func FromRanges(ranges4 []Range4, ranges6 []Range6, storage any) (*Set, error) { + for i, ipRange := range ranges4 { + if ipRange.From > ipRange.To { + return nil, E.New("ipset: malformed range ", i) + } + if i > 0 && ranges4[i-1].To >= ipRange.From { + return nil, E.New("ipset: unordered range ", i) + } + } + for i, ipRange := range ranges6 { + if compare16(ipRange.From, ipRange.To) > 0 { + return nil, E.New("ipset: malformed range ", i) + } + if i > 0 && compare16(ranges6[i-1].To, ipRange.From) >= 0 { + return nil, E.New("ipset: unordered range ", i) + } + } + return &Set{ + ranges4: ranges4, + ranges6: ranges6, + storage: storage, + }, nil +} + +func (s *Set) Ranges4() []Range4 { + return s.ranges4 +} + +func (s *Set) Ranges6() []Range6 { + return s.ranges6 +} + +func (s *Set) Contains(addr netip.Addr) bool { + if addr.Is4() { + value := binary.BigEndian.Uint32(addr.AsSlice()) + index, found := slices.BinarySearchFunc(s.ranges4, value, func(ipRange Range4, target uint32) int { + if ipRange.From > target { + return 1 + } + if ipRange.To < target { + return -1 + } + return 0 + }) + _ = index + return found + } + value := addr.As16() + _, found := slices.BinarySearchFunc(s.ranges6, value, func(ipRange Range6, target [16]byte) int { + if compare16(ipRange.From, target) > 0 { + return 1 + } + if compare16(ipRange.To, target) < 0 { + return -1 + } + return 0 + }) + return found +} + +func (s *Set) IPSet() *netipx.IPSet { + var builder netipx.IPSetBuilder + for _, ipRange := range s.ranges4 { + var from, to [4]byte + binary.BigEndian.PutUint32(from[:], ipRange.From) + binary.BigEndian.PutUint32(to[:], ipRange.To) + builder.AddRange(netipx.IPRangeFrom(netip.AddrFrom4(from), netip.AddrFrom4(to))) + } + for _, ipRange := range s.ranges6 { + builder.AddRange(netipx.IPRangeFrom(netip.AddrFrom16(ipRange.From), netip.AddrFrom16(ipRange.To))) + } + set, err := builder.IPSet() + if err != nil { + panic(err) + } + return set +} + +func (s *Set) Prefixes() []netip.Prefix { + return s.IPSet().Prefixes() +} + +func compare16(a [16]byte, b [16]byte) int { + aHigh := binary.BigEndian.Uint64(a[:8]) + bHigh := binary.BigEndian.Uint64(b[:8]) + if aHigh != bHigh { + if aHigh < bHigh { + return -1 + } + return 1 + } + aLow := binary.BigEndian.Uint64(a[8:]) + bLow := binary.BigEndian.Uint64(b[8:]) + if aLow != bLow { + if aLow < bLow { + return -1 + } + return 1 + } + return 0 +} diff --git a/common/srs/binary.go b/common/srs/binary.go index 4aac75c6b..6b48dc850 100644 --- a/common/srs/binary.go +++ b/common/srs/binary.go @@ -8,6 +8,7 @@ import ( "net/netip" "unsafe" + "github.com/sagernet/sing-box/common/ipset" C "github.com/sagernet/sing-box/constant" "github.com/sagernet/sing-box/option" "github.com/sagernet/sing/common" @@ -80,7 +81,7 @@ func Read(reader io.Reader, recover bool) (ruleSetCompat option.PlainRuleSetComp ruleSetCompat.Version = version for i := range length { var rule option.HeadlessRule - rule, err = readRule(bReader, recover, 0) + rule, err = readRule(bReader, recover, 0, nil) if err != nil { err = E.Cause(err, "read rule[", i, "]") return @@ -109,7 +110,7 @@ func Write(writer io.Writer, ruleSet option.PlainRuleSet, generateVersion uint8) return err } for _, rule := range ruleSet.Rules { - err = writeRule(bWriter, rule, generateVersion) + err = writeRule(bWriter, rule, generateVersion, nil) if err != nil { return err } @@ -123,7 +124,7 @@ func Write(writer io.Writer, ruleSet option.PlainRuleSet, generateVersion uint8) const maxLogicalRuleDepth = 100 -func readRule(reader varbin.Reader, recover bool, depth int) (rule option.HeadlessRule, err error) { +func readRule(reader varbin.Reader, recover bool, depth int, mmap *mmapReader) (rule option.HeadlessRule, err error) { if depth > maxLogicalRuleDepth { err = E.New("logical rule nested too deep") return @@ -136,28 +137,28 @@ func readRule(reader varbin.Reader, recover bool, depth int) (rule option.Headle switch ruleType { case 0: rule.Type = C.RuleTypeDefault - rule.DefaultOptions, err = readDefaultRule(reader, recover) + rule.DefaultOptions, err = readDefaultRule(reader, recover, mmap) case 1: rule.Type = C.RuleTypeLogical - rule.LogicalOptions, err = readLogicalRule(reader, recover, depth) + rule.LogicalOptions, err = readLogicalRule(reader, recover, depth, mmap) default: err = E.New("unknown rule type: ", ruleType) } return } -func writeRule(writer varbin.Writer, rule option.HeadlessRule, generateVersion uint8) error { +func writeRule(writer varbin.Writer, rule option.HeadlessRule, generateVersion uint8, mmap *mmapWriter) error { switch rule.Type { case C.RuleTypeDefault: - return writeDefaultRule(writer, rule.DefaultOptions, generateVersion) + return writeDefaultRule(writer, rule.DefaultOptions, generateVersion, mmap) case C.RuleTypeLogical: - return writeLogicalRule(writer, rule.LogicalOptions, generateVersion) + return writeLogicalRule(writer, rule.LogicalOptions, generateVersion, mmap) default: panic("unknown rule type: " + rule.Type) } } -func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHeadlessRule, err error) { +func readDefaultRule(reader varbin.Reader, recover bool, mmap *mmapReader) (rule option.DefaultHeadlessRule, err error) { var lastItemType uint8 for { var itemType uint8 @@ -179,7 +180,11 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea rule.Network, err = readRuleItemString(reader) case ruleItemDomain: var matcher *domain.Matcher - matcher, err = domain.ReadMatcher(reader) + if mmap != nil { + matcher, err = mmap.readMatcher(reader) + } else { + matcher, err = domain.ReadMatcher(reader) + } if err != nil { return } @@ -192,7 +197,11 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea case ruleItemDomainRegex: rule.DomainRegex, err = readRuleItemString(reader) case ruleItemSourceIPCIDR: - rule.SourceIPSet, err = readIPSet(reader) + if mmap != nil { + rule.SourceIPSet, err = mmap.readIPSet(reader) + } else { + rule.SourceIPSet, err = readIPSet(reader) + } if err != nil { return } @@ -200,7 +209,11 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea rule.SourceIPCIDR = common.Map(rule.SourceIPSet.Prefixes(), netip.Prefix.String) } case ruleItemIPCIDR: - rule.IPSet, err = readIPSet(reader) + if mmap != nil { + rule.IPSet, err = mmap.readIPSet(reader) + } else { + rule.IPSet, err = readIPSet(reader) + } if err != nil { return } @@ -231,7 +244,11 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea rule.WIFIBSSID, err = readRuleItemString(reader) case ruleItemAdGuardDomain: var matcher *domain.AdGuardMatcher - matcher, err = domain.ReadAdGuardMatcher(reader) + if mmap != nil { + matcher, err = mmap.readAdGuardMatcher(reader) + } else { + matcher, err = domain.ReadAdGuardMatcher(reader) + } if err != nil { return } @@ -303,7 +320,7 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea } } -func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, generateVersion uint8) error { +func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, generateVersion uint8, mmap *mmapWriter) error { err := binary.Write(writer, binary.BigEndian, uint8(0)) if err != nil { return err @@ -322,12 +339,20 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen return err } } - if len(rule.Domain) > 0 || len(rule.DomainSuffix) > 0 { + if len(rule.Domain) > 0 || len(rule.DomainSuffix) > 0 || mmap != nil && rule.DomainMatcher != nil { err = binary.Write(writer, binary.BigEndian, ruleItemDomain) if err != nil { return err } - err = domain.NewMatcher(rule.Domain, rule.DomainSuffix, generateVersion == C.RuleSetVersion1).Write(writer) + matcher := rule.DomainMatcher + if len(rule.Domain) > 0 || len(rule.DomainSuffix) > 0 { + matcher = domain.NewMatcher(rule.Domain, rule.DomainSuffix, generateVersion == C.RuleSetVersion1) + } + if mmap != nil { + err = mmap.writeMatcher(writer, matcher) + } else { + err = matcher.Write(writer) + } if err != nil { return err } @@ -344,14 +369,14 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen return err } } - if len(rule.SourceIPCIDR) > 0 { - err = writeRuleItemCIDR(writer, ruleItemSourceIPCIDR, rule.SourceIPCIDR) + if len(rule.SourceIPCIDR) > 0 || mmap != nil && rule.SourceIPSet != nil { + err = writeRuleItemCIDR(writer, ruleItemSourceIPCIDR, rule.SourceIPCIDR, rule.SourceIPSet, mmap) if err != nil { return E.Cause(err, "source_ip_cidr") } } - if len(rule.IPCIDR) > 0 { - err = writeRuleItemCIDR(writer, ruleItemIPCIDR, rule.IPCIDR) + if len(rule.IPCIDR) > 0 || mmap != nil && rule.IPSet != nil { + err = writeRuleItemCIDR(writer, ruleItemIPCIDR, rule.IPCIDR, rule.IPSet, mmap) if err != nil { return E.Cause(err, "ipcidr") } @@ -500,7 +525,7 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen return err } } - if len(rule.AdGuardDomain) > 0 { + if len(rule.AdGuardDomain) > 0 || mmap != nil && rule.AdGuardDomainMatcher != nil { if generateVersion < C.RuleSetVersion2 { return E.New("AdGuard rule items is only supported in version 2 or later") } @@ -508,7 +533,15 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen if err != nil { return err } - err = domain.NewAdGuardMatcher(rule.AdGuardDomain).Write(writer) + matcher := rule.AdGuardDomainMatcher + if len(rule.AdGuardDomain) > 0 { + matcher = domain.NewAdGuardMatcher(rule.AdGuardDomain) + } + if mmap != nil { + err = mmap.writeAdGuardMatcher(writer, matcher) + } else { + err = matcher.Write(writer) + } if err != nil { return err } @@ -588,33 +621,40 @@ func writeRuleItemUint16(writer varbin.Writer, itemType uint8, value []uint16) e return binary.Write(writer, binary.BigEndian, value) } -func writeRuleItemCIDR(writer varbin.Writer, itemType uint8, value []string) error { - var builder netipx.IPSetBuilder - for i, prefixString := range value { - prefix, err := netip.ParsePrefix(prefixString) - if err == nil { - builder.AddPrefix(prefix) - continue +func writeRuleItemCIDR(writer varbin.Writer, itemType uint8, value []string, rawSet *ipset.Set, mmap *mmapWriter) error { + set := rawSet + if len(value) > 0 { + var builder netipx.IPSetBuilder + for i, prefixString := range value { + prefix, err := netip.ParsePrefix(prefixString) + if err == nil { + builder.AddPrefix(prefix) + continue + } + addr, addrErr := netip.ParseAddr(prefixString) + if addrErr == nil { + builder.Add(addr) + continue + } + return E.Cause(err, "parse [", i, "]") } - addr, addrErr := netip.ParseAddr(prefixString) - if addrErr == nil { - builder.Add(addr) - continue + ipSet, err := builder.IPSet() + if err != nil { + return err } - return E.Cause(err, "parse [", i, "]") + set = ipset.FromIPSet(ipSet) } - ipSet, err := builder.IPSet() + err := binary.Write(writer, binary.BigEndian, itemType) if err != nil { return err } - err = binary.Write(writer, binary.BigEndian, itemType) - if err != nil { - return err + if mmap != nil { + return mmap.writeIPSet(writer, set) } - return writeIPSet(writer, ipSet) + return writeIPSet(writer, set) } -func readLogicalRule(reader varbin.Reader, recovery bool, depth int) (logicalRule option.LogicalHeadlessRule, err error) { +func readLogicalRule(reader varbin.Reader, recovery bool, depth int, mmap *mmapReader) (logicalRule option.LogicalHeadlessRule, err error) { mode, err := reader.ReadByte() if err != nil { return @@ -634,7 +674,7 @@ func readLogicalRule(reader varbin.Reader, recovery bool, depth int) (logicalRul } for i := range length { var rule option.HeadlessRule - rule, err = readRule(reader, recovery, depth+1) + rule, err = readRule(reader, recovery, depth+1, mmap) if err != nil { err = E.Cause(err, "read logical rule [", i, "]") return @@ -648,7 +688,7 @@ func readLogicalRule(reader varbin.Reader, recovery bool, depth int) (logicalRul return } -func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRule, generateVersion uint8) error { +func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRule, generateVersion uint8, mmap *mmapWriter) error { err := binary.Write(writer, binary.BigEndian, uint8(1)) if err != nil { return err @@ -669,7 +709,7 @@ func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRu return err } for _, rule := range logicalRule.Rules { - err = writeRule(writer, rule, generateVersion) + err = writeRule(writer, rule, generateVersion, mmap) if err != nil { return err } diff --git a/common/srs/compat_test.go b/common/srs/compat_test.go index fdac6be80..cd52ff126 100644 --- a/common/srs/compat_test.go +++ b/common/srs/compat_test.go @@ -9,6 +9,7 @@ import ( "testing" "unsafe" + "github.com/sagernet/sing-box/common/ipset" M "github.com/sagernet/sing/common/metadata" "github.com/sagernet/sing/common/varbin" @@ -94,14 +95,11 @@ func oldReadIPSet(reader varbin.Reader) (*netipx.IPSet, error) { if err != nil { return nil, err } - mySet := &myIPSet{ - rr: make([]myIPRange, len(ranges)), + var builder netipx.IPSetBuilder + for _, rangeData := range ranges { + builder.AddRange(netipx.IPRangeFrom(M.AddrFromIP(rangeData.From), M.AddrFromIP(rangeData.To))) } - for i, rangeData := range ranges { - mySet.rr[i].from = M.AddrFromIP(rangeData.From) - mySet.rr[i].to = M.AddrFromIP(rangeData.To) - } - return (*netipx.IPSet)(unsafe.Pointer(mySet)), nil + return builder.IPSet() } // New write functions (without itemType prefix for testing) @@ -384,7 +382,7 @@ func TestIPSetCompat(t *testing.T) { // New write var newBuf bytes.Buffer - err := writeIPSet(&newBuf, tc.input) + err := writeIPSet(&newBuf, ipset.FromIPSet(tc.input)) require.NoError(t, err) // Verify format starts with version byte (1) + uint64 count @@ -399,7 +397,7 @@ func TestIPSetCompat(t *testing.T) { // New write -> new read readBack2, err := readIPSet(bufio.NewReader(bytes.NewReader(newBuf.Bytes()))) require.NoError(t, err) - requireIPSetEqual(t, tc.input, readBack2) + requireIPSetEqual(t, tc.input, readBack2.IPSet()) }) } } diff --git a/common/srs/ip_set.go b/common/srs/ip_set.go index 01def6bb2..6df0c336d 100644 --- a/common/srs/ip_set.go +++ b/common/srs/ip_set.go @@ -3,26 +3,14 @@ package srs import ( "encoding/binary" "io" - "net/netip" "os" - "unsafe" + "slices" - M "github.com/sagernet/sing/common/metadata" + "github.com/sagernet/sing-box/common/ipset" "github.com/sagernet/sing/common/varbin" - - "go4.org/netipx" ) -type myIPSet struct { - rr []myIPRange -} - -type myIPRange struct { - from netip.Addr - to netip.Addr -} - -func readIPSet(reader varbin.Reader) (*netipx.IPSet, error) { +func readIPSet(reader varbin.Reader) (*ipset.Set, error) { version, err := reader.ReadByte() if err != nil { return nil, err @@ -36,67 +24,104 @@ func readIPSet(reader varbin.Reader) (*netipx.IPSet, error) { if err != nil { return nil, err } - mySet := &myIPSet{} + var ( + ranges4 []ipset.Range4 + ranges6 []ipset.Range6 + ) for range length { - var from, to netip.Addr - from, err = readIPSetAddr(reader) + var from, to [16]byte + var fromLength, toLength int + fromLength, err = readIPSetAddr(reader, &from) if err != nil { return nil, err } - to, err = readIPSetAddr(reader) + toLength, err = readIPSetAddr(reader, &to) if err != nil { return nil, err } - mySet.rr = append(mySet.rr, myIPRange{from: from, to: to}) + if fromLength != toLength { + return nil, os.ErrInvalid + } + if fromLength == 4 { + ranges4 = append(ranges4, ipset.Range4{ + From: binary.BigEndian.Uint32(from[:4]), + To: binary.BigEndian.Uint32(to[:4]), + }) + } else { + ranges6 = append(ranges6, ipset.Range6{From: from, To: to}) + } } - return (*netipx.IPSet)(unsafe.Pointer(mySet)), nil + slices.SortFunc(ranges4, func(a, b ipset.Range4) int { + if a.From < b.From { + return -1 + } else if a.From > b.From { + return 1 + } + return 0 + }) + slices.SortFunc(ranges6, func(a, b ipset.Range6) int { + return slices.Compare(a.From[:], b.From[:]) + }) + return ipset.FromRanges(slices.Clip(ranges4), slices.Clip(ranges6), nil) } -func readIPSetAddr(reader varbin.Reader) (netip.Addr, error) { +func readIPSetAddr(reader varbin.Reader, addr *[16]byte) (int, error) { addrLen, err := binary.ReadUvarint(reader) if err != nil { - return netip.Addr{}, err + return 0, err } if addrLen != 4 && addrLen != 16 { - return netip.Addr{}, os.ErrInvalid + return 0, os.ErrInvalid } - var addrBytes [16]byte - _, err = io.ReadFull(reader, addrBytes[:addrLen]) + _, err = io.ReadFull(reader, addr[:addrLen]) if err != nil { - return netip.Addr{}, err + return 0, err } - return M.AddrFromIP(addrBytes[:addrLen]), nil + return int(addrLen), nil } -func writeIPSet(writer varbin.Writer, set *netipx.IPSet) error { +func writeIPSet(writer varbin.Writer, set *ipset.Set) error { err := writer.WriteByte(1) if err != nil { return err } - mySet := (*myIPSet)(unsafe.Pointer(set)) - err = binary.Write(writer, binary.BigEndian, uint64(len(mySet.rr))) + ranges4 := set.Ranges4() + ranges6 := set.Ranges6() + err = binary.Write(writer, binary.BigEndian, uint64(len(ranges4)+len(ranges6))) if err != nil { return err } - for _, rr := range mySet.rr { - fromBytes := rr.from.AsSlice() - _, err = varbin.WriteUvarint(writer, uint64(len(fromBytes))) + for _, ipRange := range ranges4 { + var from, to [4]byte + binary.BigEndian.PutUint32(from[:], ipRange.From) + binary.BigEndian.PutUint32(to[:], ipRange.To) + err = writeIPSetAddr(writer, from[:]) if err != nil { return err } - _, err = writer.Write(fromBytes) + err = writeIPSetAddr(writer, to[:]) if err != nil { return err } - toBytes := rr.to.AsSlice() - _, err = varbin.WriteUvarint(writer, uint64(len(toBytes))) + } + for _, ipRange := range ranges6 { + err = writeIPSetAddr(writer, ipRange.From[:]) if err != nil { return err } - _, err = writer.Write(toBytes) + err = writeIPSetAddr(writer, ipRange.To[:]) if err != nil { return err } } return nil } + +func writeIPSetAddr(writer varbin.Writer, addr []byte) error { + _, err := varbin.WriteUvarint(writer, uint64(len(addr))) + if err != nil { + return err + } + _, err = writer.Write(addr) + return err +} diff --git a/common/srs/mmap.go b/common/srs/mmap.go new file mode 100644 index 000000000..7fbe84c8b --- /dev/null +++ b/common/srs/mmap.go @@ -0,0 +1,403 @@ +package srs + +import ( + "bufio" + "bytes" + "encoding/binary" + "io" + "os" + "runtime" + "unsafe" + + "github.com/sagernet/sing-box/common/ipset" + "github.com/sagernet/sing-box/option" + "github.com/sagernet/sing/common/domain" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/varbin" +) + +var mmapMagic = [4]byte{0x53, 0x52, 0x53, 0x4D} // SRSM + +const ( + mmapFormatVersion = 1 + mmapHeaderSize = 40 + mmapBlobEntrySize = 24 + mmapBlobSuccinct = 1 + mmapBlobIPSet = 2 +) + +type mmapStorage struct { + release func() +} + +type mmapBlob struct { + kind uint64 + matcher domain.Mmap + set *ipset.Set +} + +type mmapWriter struct { + blobs []mmapBlob +} + +func (w *mmapWriter) writeMatcher(writer varbin.Writer, matcher *domain.Matcher) error { + w.blobs = append(w.blobs, mmapBlob{kind: mmapBlobSuccinct, matcher: matcher.Mmap()}) + _, err := varbin.WriteUvarint(writer, uint64(len(w.blobs)-1)) + return err +} + +func (w *mmapWriter) writeAdGuardMatcher(writer varbin.Writer, matcher *domain.AdGuardMatcher) error { + w.blobs = append(w.blobs, mmapBlob{kind: mmapBlobSuccinct, matcher: matcher.Mmap()}) + _, err := varbin.WriteUvarint(writer, uint64(len(w.blobs)-1)) + return err +} + +func (w *mmapWriter) writeIPSet(writer varbin.Writer, set *ipset.Set) error { + w.blobs = append(w.blobs, mmapBlob{kind: mmapBlobIPSet, set: set}) + _, err := varbin.WriteUvarint(writer, uint64(len(w.blobs)-1)) + return err +} + +func WriteMmap(file io.WriteSeeker, ruleSet option.PlainRuleSetCompat) error { + var header [mmapHeaderSize]byte + _, err := file.Write(header[:]) + if err != nil { + return err + } + mmap := &mmapWriter{} + writer := bufio.NewWriter(file) + _, err = varbin.WriteUvarint(writer, uint64(len(ruleSet.Options.Rules))) + if err != nil { + return err + } + for _, rule := range ruleSet.Options.Rules { + err = writeRule(writer, rule, ruleSet.Version, mmap) + if err != nil { + return err + } + } + err = writer.Flush() + if err != nil { + return err + } + rulesEnd, err := file.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + rulesLength := uint64(rulesEnd) - mmapHeaderSize + blobTableOffset := alignMmapOffset(uint64(rulesEnd)) + _, err = writer.Write(make([]byte, blobTableOffset-uint64(rulesEnd))) + if err != nil { + return err + } + blobOffset := blobTableOffset + uint64(len(mmap.blobs))*mmapBlobEntrySize + for _, blob := range mmap.blobs { + length := blob.size() + err = writeMmapUint64(writer, blob.kind, blobOffset, length) + if err != nil { + return err + } + blobOffset += length + } + for _, blob := range mmap.blobs { + err = blob.write(writer) + if err != nil { + return err + } + } + err = writer.Flush() + if err != nil { + return err + } + copy(header[0:4], mmapMagic[:]) + binary.NativeEndian.PutUint32(header[4:8], mmapFormatVersion) + binary.NativeEndian.PutUint64(header[8:16], uint64(ruleSet.Version)) + binary.NativeEndian.PutUint64(header[16:24], rulesLength) + binary.NativeEndian.PutUint64(header[24:32], blobTableOffset) + binary.NativeEndian.PutUint64(header[32:40], uint64(len(mmap.blobs))) + _, err = file.Seek(0, io.SeekStart) + if err != nil { + return err + } + _, err = file.Write(header[:]) + return err +} + +func (b *mmapBlob) size() uint64 { + switch b.kind { + case mmapBlobSuccinct: + return 40 + + uint64(len(b.matcher.Leaves))*8 + + uint64(len(b.matcher.LabelBitmap))*8 + + alignMmapOffset(uint64(len(b.matcher.Labels))) + + alignMmapOffset(uint64(len(b.matcher.Ranks))*4) + + alignMmapOffset(uint64(len(b.matcher.Selects))*4) + case mmapBlobIPSet: + return 16 + + uint64(len(b.set.Ranges4()))*8 + + uint64(len(b.set.Ranges6()))*32 + default: + panic("unknown mmap blob kind") + } +} + +func (b *mmapBlob) write(writer *bufio.Writer) error { + switch b.kind { + case mmapBlobSuccinct: + err := writeMmapUint64(writer, + uint64(len(b.matcher.Leaves)), + uint64(len(b.matcher.LabelBitmap)), + uint64(len(b.matcher.Labels)), + uint64(len(b.matcher.Ranks)), + uint64(len(b.matcher.Selects)), + ) + if err != nil { + return err + } + err = writeMmapArray(writer, b.matcher.Leaves) + if err != nil { + return err + } + err = writeMmapArray(writer, b.matcher.LabelBitmap) + if err != nil { + return err + } + err = writeMmapArray(writer, b.matcher.Labels) + if err != nil { + return err + } + err = writeMmapArray(writer, b.matcher.Ranks) + if err != nil { + return err + } + return writeMmapArray(writer, b.matcher.Selects) + case mmapBlobIPSet: + err := writeMmapUint64(writer, uint64(len(b.set.Ranges4())), uint64(len(b.set.Ranges6()))) + if err != nil { + return err + } + err = writeMmapArray(writer, b.set.Ranges4()) + if err != nil { + return err + } + return writeMmapArray(writer, b.set.Ranges6()) + default: + panic("unknown mmap blob kind") + } +} + +func writeMmapUint64(writer *bufio.Writer, values ...uint64) error { + var buffer [8]byte + for _, value := range values { + binary.NativeEndian.PutUint64(buffer[:], value) + _, err := writer.Write(buffer[:]) + if err != nil { + return err + } + } + return nil +} + +func writeMmapArray[T any](writer *bufio.Writer, values []T) error { + if len(values) == 0 { + return nil + } + length := uintptr(len(values)) * unsafe.Sizeof(values[0]) + _, err := writer.Write(unsafe.Slice((*byte)(unsafe.Pointer(&values[0])), length)) + if err != nil { + return err + } + padding := alignMmapOffset(uint64(length)) - uint64(length) + if padding > 0 { + var zero [8]byte + _, err = writer.Write(zero[:padding]) + } + return err +} + +func alignMmapOffset(offset uint64) uint64 { + return (offset + 7) &^ 7 +} + +type mmapReader struct { + data []byte + storage *mmapStorage + blobTable []byte + blobCount uint64 +} + +func ReadMmap(file *os.File) (ruleSet option.PlainRuleSetCompat, err error) { + stat, err := file.Stat() + if err != nil { + return + } + if stat.Size() < mmapHeaderSize { + err = E.New("truncated rule-set mmap") + return + } + data, release, err := mmapFile(file, int(stat.Size())) + if err != nil { + return + } + storage := &mmapStorage{release: release} + cleanup := runtime.AddCleanup(storage, func(release func()) { release() }, release) + defer func() { + if err != nil { + cleanup.Stop() + release() + } + }() + if [4]byte(data[0:4]) != mmapMagic { + err = E.New("invalid rule-set mmap") + return + } + if binary.NativeEndian.Uint32(data[4:8]) != mmapFormatVersion { + err = E.New("unsupported rule-set mmap version") + return + } + rulesLength := binary.NativeEndian.Uint64(data[16:24]) + blobTableOffset := binary.NativeEndian.Uint64(data[24:32]) + blobCount := binary.NativeEndian.Uint64(data[32:40]) + rules, err := mmapSection(data, mmapHeaderSize, rulesLength) + if err != nil { + return + } + blobTable, err := mmapSection(data, blobTableOffset, blobCount*mmapBlobEntrySize) + if err != nil { + return + } + mmap := &mmapReader{ + data: data, + storage: storage, + blobTable: blobTable, + blobCount: blobCount, + } + reader := bytes.NewReader(rules) + ruleSet.Version = uint8(binary.NativeEndian.Uint64(data[8:16])) + length, err := binary.ReadUvarint(reader) + if err != nil { + return + } + for i := range length { + var rule option.HeadlessRule + rule, err = readRule(reader, false, 0, mmap) + if err != nil { + err = E.Cause(err, "read rule[", i, "]") + return + } + ruleSet.Options.Rules = append(ruleSet.Options.Rules, rule) + } + return +} + +func mmapSection(data []byte, offset uint64, length uint64) ([]byte, error) { + if offset%8 != 0 || offset > uint64(len(data)) || length > uint64(len(data))-offset { + return nil, E.New("rule-set mmap section out of bounds") + } + return data[offset : offset+length : offset+length], nil +} + +func (r *mmapReader) blob(reader varbin.Reader, kind uint64) ([]byte, error) { + index, err := binary.ReadUvarint(reader) + if err != nil { + return nil, err + } + if index >= r.blobCount { + return nil, E.New("rule-set mmap blob index out of range") + } + entry := r.blobTable[index*mmapBlobEntrySize:] + if binary.NativeEndian.Uint64(entry[0:8]) != kind { + return nil, E.New("rule-set mmap blob kind mismatch") + } + return mmapSection(r.data, binary.NativeEndian.Uint64(entry[8:16]), binary.NativeEndian.Uint64(entry[16:24])) +} + +func (r *mmapReader) readMatcher(reader varbin.Reader) (*domain.Matcher, error) { + mmap, err := r.readSuccinctMmap(reader) + if err != nil { + return nil, err + } + return domain.NewMatcherFromMmap(mmap) +} + +func (r *mmapReader) readAdGuardMatcher(reader varbin.Reader) (*domain.AdGuardMatcher, error) { + mmap, err := r.readSuccinctMmap(reader) + if err != nil { + return nil, err + } + return domain.NewAdGuardMatcherFromMmap(mmap) +} + +func (r *mmapReader) readSuccinctMmap(reader varbin.Reader) (mmap domain.Mmap, err error) { + blob, err := r.blob(reader, mmapBlobSuccinct) + if err != nil { + return + } + if len(blob) < 40 { + err = E.New("rule-set mmap blob truncated") + return + } + var counts [5]uint64 + for i := range counts { + counts[i] = binary.NativeEndian.Uint64(blob[i*8:]) + } + cursor := blob[40:] + mmap.Leaves, cursor, err = mmapArray[uint64](cursor, counts[0]) + if err != nil { + return + } + mmap.LabelBitmap, cursor, err = mmapArray[uint64](cursor, counts[1]) + if err != nil { + return + } + mmap.Labels, cursor, err = mmapArray[byte](cursor, counts[2]) + if err != nil { + return + } + mmap.Ranks, cursor, err = mmapArray[int32](cursor, counts[3]) + if err != nil { + return + } + mmap.Selects, _, err = mmapArray[int32](cursor, counts[4]) + if err != nil { + return + } + mmap.Storage = r.storage + return +} + +func (r *mmapReader) readIPSet(reader varbin.Reader) (*ipset.Set, error) { + blob, err := r.blob(reader, mmapBlobIPSet) + if err != nil { + return nil, err + } + if len(blob) < 16 { + return nil, E.New("rule-set mmap blob truncated") + } + cursor := blob[16:] + ranges4, cursor, err := mmapArray[ipset.Range4](cursor, binary.NativeEndian.Uint64(blob[0:8])) + if err != nil { + return nil, err + } + ranges6, _, err := mmapArray[ipset.Range6](cursor, binary.NativeEndian.Uint64(blob[8:16])) + if err != nil { + return nil, err + } + return ipset.FromRanges(ranges4, ranges6, r.storage) +} + +func mmapArray[T any](data []byte, count uint64) ([]T, []byte, error) { + var zero T + size := uint64(unsafe.Sizeof(zero)) + if count > uint64(len(data))/size { + return nil, nil, E.New("rule-set mmap array out of bounds") + } + length := alignMmapOffset(count * size) + if length > uint64(len(data)) { + return nil, nil, E.New("rule-set mmap array out of bounds") + } + if count == 0 { + return nil, data[length:], nil + } + return unsafe.Slice((*T)(unsafe.Pointer(&data[0])), count), data[length:], nil +} diff --git a/common/srs/mmap_stub.go b/common/srs/mmap_stub.go new file mode 100644 index 000000000..3507c3263 --- /dev/null +++ b/common/srs/mmap_stub.go @@ -0,0 +1,13 @@ +//go:build !unix + +package srs + +import ( + "os" + + E "github.com/sagernet/sing/common/exceptions" +) + +func mmapFile(file *os.File, size int) ([]byte, func(), error) { + return nil, nil, E.New("rule-set mmap is not supported on this platform") +} diff --git a/common/srs/mmap_test.go b/common/srs/mmap_test.go new file mode 100644 index 000000000..f18f6cd0c --- /dev/null +++ b/common/srs/mmap_test.go @@ -0,0 +1,80 @@ +//go:build unix + +package srs + +import ( + "bytes" + "io" + "net/netip" + "os" + "path/filepath" + "testing" + + C "github.com/sagernet/sing-box/constant" + "github.com/sagernet/sing-box/option" + "github.com/sagernet/sing/common/json/badoption" + + "github.com/stretchr/testify/require" +) + +func TestMmapMatchesSource(t *testing.T) { + t.Parallel() + source := option.PlainRuleSet{Rules: []option.HeadlessRule{ + { + Type: C.RuleTypeDefault, + DefaultOptions: option.DefaultHeadlessRule{ + Domain: badoption.Listable[string]{"example.com"}, + DomainSuffix: badoption.Listable[string]{".example.org"}, + IPCIDR: badoption.Listable[string]{"10.0.0.0/8", "2001:db8::/32", "192.0.2.1"}, + }, + }, + { + Type: C.RuleTypeLogical, + LogicalOptions: option.LogicalHeadlessRule{ + Mode: C.LogicalTypeOr, + Rules: []option.HeadlessRule{{ + Type: C.RuleTypeDefault, + DefaultOptions: option.DefaultHeadlessRule{ + AdGuardDomain: badoption.Listable[string]{"||ads.example.net^"}, + SourceIPCIDR: badoption.Listable[string]{"172.16.0.0/12"}, + }, + }}, + }, + }, + }} + var binary bytes.Buffer + require.NoError(t, Write(&binary, source, C.RuleSetVersionCurrent)) + parsed, err := Read(bytes.NewReader(binary.Bytes()), false) + require.NoError(t, err) + + mmapPath := filepath.Join(t.TempDir(), "rule-set.mmap") + mmapFile, err := os.Create(mmapPath) + require.NoError(t, err) + defer mmapFile.Close() + require.NoError(t, WriteMmap(mmapFile, parsed)) + _, err = mmapFile.Seek(0, io.SeekStart) + require.NoError(t, err) + mmap, err := ReadMmap(mmapFile) + require.NoError(t, err) + _, err = os.Stat(mmapPath) + require.ErrorIs(t, err, os.ErrNotExist) + require.Equal(t, parsed.Version, mmap.Version) + require.Len(t, mmap.Options.Rules, 2) + + first := mmap.Options.Rules[0].DefaultOptions + require.True(t, first.DomainMatcher.Match("example.com")) + require.True(t, first.DomainMatcher.Match("www.example.org")) + require.False(t, first.DomainMatcher.Match("example.org")) + require.False(t, first.DomainMatcher.Match("example.net")) + require.True(t, first.IPSet.Contains(netip.MustParseAddr("10.255.0.1"))) + require.True(t, first.IPSet.Contains(netip.MustParseAddr("192.0.2.1"))) + require.False(t, first.IPSet.Contains(netip.MustParseAddr("192.0.2.2"))) + require.True(t, first.IPSet.Contains(netip.MustParseAddr("2001:db8::1"))) + require.False(t, first.IPSet.Contains(netip.MustParseAddr("2001:db9::1"))) + + second := mmap.Options.Rules[1].LogicalOptions.Rules[0].DefaultOptions + require.True(t, second.AdGuardDomainMatcher.Match("ads.example.net")) + require.False(t, second.AdGuardDomainMatcher.Match("example.net")) + require.True(t, second.SourceIPSet.Contains(netip.MustParseAddr("172.31.255.255"))) + require.False(t, second.SourceIPSet.Contains(netip.MustParseAddr("172.32.0.0"))) +} diff --git a/common/srs/mmap_unix.go b/common/srs/mmap_unix.go new file mode 100644 index 000000000..2d76ecf7e --- /dev/null +++ b/common/srs/mmap_unix.go @@ -0,0 +1,20 @@ +//go:build unix + +package srs + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func mmapFile(file *os.File, size int) ([]byte, func(), error) { + data, err := unix.Mmap(int(file.Fd()), 0, size, unix.PROT_READ, unix.MAP_PRIVATE) + if err != nil { + return nil, nil, err + } + os.Remove(file.Name()) + return data, func() { + unix.Munmap(data) + }, nil +} diff --git a/option/rule_set.go b/option/rule_set.go index a14274bb0..305de0dd2 100644 --- a/option/rule_set.go +++ b/option/rule_set.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" + "github.com/sagernet/sing-box/common/ipset" C "github.com/sagernet/sing-box/constant" "github.com/sagernet/sing-box/schema" "github.com/sagernet/sing/common" @@ -15,8 +16,6 @@ import ( "github.com/sagernet/sing/common/json" "github.com/sagernet/sing/common/json/badjson" "github.com/sagernet/sing/common/json/badoption" - - "go4.org/netipx" ) type _RuleSet struct { @@ -277,8 +276,8 @@ type DefaultHeadlessRule struct { Invert bool `json:"invert,omitempty"` DomainMatcher *domain.Matcher `json:"-"` - SourceIPSet *netipx.IPSet `json:"-"` - IPSet *netipx.IPSet `json:"-"` + SourceIPSet *ipset.Set `json:"-"` + IPSet *ipset.Set `json:"-"` AdGuardDomain badoption.Listable[string] `json:"-"` AdGuardDomainMatcher *domain.AdGuardMatcher `json:"-"` diff --git a/route/rule/rule_item_cidr.go b/route/rule/rule_item_cidr.go index 9780a8ec4..8c27994c2 100644 --- a/route/rule/rule_item_cidr.go +++ b/route/rule/rule_item_cidr.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/sagernet/sing-box/adapter" + "github.com/sagernet/sing-box/common/ipset" "github.com/sagernet/sing/common" E "github.com/sagernet/sing/common/exceptions" @@ -15,7 +16,7 @@ import ( var _ RuleItem = (*IPCIDRItem)(nil) type IPCIDRItem struct { - ipSet *netipx.IPSet + ipSet *ipset.Set isSource bool description string } @@ -53,13 +54,13 @@ func NewIPCIDRItem(isSource bool, prefixStrings []string) (*IPCIDRItem, error) { return nil, err } return &IPCIDRItem{ - ipSet: ipSet, + ipSet: ipset.FromIPSet(ipSet), isSource: isSource, description: description, }, nil } -func NewRawIPCIDRItem(isSource bool, ipSet *netipx.IPSet) *IPCIDRItem { +func NewRawIPCIDRItem(isSource bool, ipSet *ipset.Set) *IPCIDRItem { var description string if isSource { description = "source_ip_cidr=" diff --git a/route/rule/rule_set.go b/route/rule/rule_set.go index 4eb24654d..13bd67a60 100644 --- a/route/rule/rule_set.go +++ b/route/rule/rule_set.go @@ -32,7 +32,7 @@ func extractIPSetFromRule(rawRule adapter.HeadlessRule) []*netipx.IPSet { return common.FlatMap(rule.destinationIPCIDRItems, func(rawItem RuleItem) []*netipx.IPSet { switch item := rawItem.(type) { case *IPCIDRItem: - return []*netipx.IPSet{item.ipSet} + return []*netipx.IPSet{item.ipSet.IPSet()} default: return nil } diff --git a/route/rule/rule_set_local.go b/route/rule/rule_set_local.go index 9d5d21e83..76699015e 100644 --- a/route/rule/rule_set_local.go +++ b/route/rule/rule_set_local.go @@ -107,7 +107,6 @@ func (s *LocalRuleSet) reloadFile(path string) error { if err != nil { return err } - case C.RuleSetFormatBinary: setFile, err := filemanager.Open(s.ctx, path) if err != nil { @@ -121,7 +120,7 @@ func (s *LocalRuleSet) reloadFile(path string) error { default: return E.New("unknown rule-set format: ", s.fileFormat) } - plainRuleSet, err := ruleSet.Upgrade() + plainRuleSet, err := mmapRuleSet(s.ctx, s.logger, s.tag, ruleSet).Upgrade() if err != nil { return err } diff --git a/route/rule/rule_set_mmap_ios.go b/route/rule/rule_set_mmap_ios.go new file mode 100644 index 000000000..eb6a03e61 --- /dev/null +++ b/route/rule/rule_set_mmap_ios.go @@ -0,0 +1,44 @@ +//go:build ios + +package rule + +import ( + "context" + "io" + "os" + + "github.com/sagernet/sing-box/common/srs" + "github.com/sagernet/sing-box/option" + E "github.com/sagernet/sing/common/exceptions" + "github.com/sagernet/sing/common/logger" + "github.com/sagernet/sing/service/filemanager" +) + +func mmapRuleSet(ctx context.Context, logger logger.Logger, tag string, ruleSet option.PlainRuleSetCompat) option.PlainRuleSetCompat { + file, err := filemanager.CreateTemp(ctx, "rule-set-*.mmap") + if err != nil { + logger.Warn(E.Cause(err, "create rule-set mmap file ", tag)) + return ruleSet + } + defer file.Close() + mmap, err := writeAndMmapRuleSet(file, ruleSet) + if err != nil { + filemanager.Remove(ctx, file.Name()) + logger.Warn(E.Cause(err, "mmap rule-set ", tag)) + return ruleSet + } + logger.Debug("mmap rule-set ", tag) + return mmap +} + +func writeAndMmapRuleSet(file *os.File, ruleSet option.PlainRuleSetCompat) (option.PlainRuleSetCompat, error) { + err := srs.WriteMmap(file, ruleSet) + if err != nil { + return ruleSet, err + } + _, err = file.Seek(0, io.SeekStart) + if err != nil { + return ruleSet, err + } + return srs.ReadMmap(file) +} diff --git a/route/rule/rule_set_mmap_stub.go b/route/rule/rule_set_mmap_stub.go new file mode 100644 index 000000000..328920c8d --- /dev/null +++ b/route/rule/rule_set_mmap_stub.go @@ -0,0 +1,14 @@ +//go:build !ios + +package rule + +import ( + "context" + + "github.com/sagernet/sing-box/option" + "github.com/sagernet/sing/common/logger" +) + +func mmapRuleSet(ctx context.Context, logger logger.Logger, tag string, ruleSet option.PlainRuleSetCompat) option.PlainRuleSetCompat { + return ruleSet +} diff --git a/route/rule/rule_set_remote.go b/route/rule/rule_set_remote.go index 10b1f13c1..5d694d81f 100644 --- a/route/rule/rule_set_remote.go +++ b/route/rule/rule_set_remote.go @@ -197,7 +197,7 @@ func (s *RemoteRuleSet) loadBytes(content []byte) error { default: return E.New("unknown rule-set format: ", s.options.Format) } - plainRuleSet, err := ruleSet.Upgrade() + plainRuleSet, err := mmapRuleSet(s.ctx, s.logger, s.tag, ruleSet).Upgrade() if err != nil { return err } diff --git a/route/rule/rule_set_update_validation_test.go b/route/rule/rule_set_update_validation_test.go index bd409095f..9c2cf10ff 100644 --- a/route/rule/rule_set_update_validation_test.go +++ b/route/rule/rule_set_update_validation_test.go @@ -2,6 +2,7 @@ package rule import ( "context" + "os" "sync/atomic" "testing" @@ -10,8 +11,10 @@ import ( "github.com/sagernet/sing-box/option" E "github.com/sagernet/sing/common/exceptions" "github.com/sagernet/sing/common/json/badoption" + "github.com/sagernet/sing/common/logger" "github.com/sagernet/sing/common/x/list" "github.com/sagernet/sing/service" + "github.com/sagernet/sing/service/filemanager" "github.com/stretchr/testify/require" ) @@ -85,9 +88,11 @@ func TestRemoteRuleSetLoadBytesRejectsInvalidUpdateBeforeCommit(t *testing.T) { return nil }, }) + ctx = filemanager.WithDefault(ctx, "", t.TempDir(), os.Getuid(), os.Getgid()) ruleSet := &RemoteRuleSet{ - ctx: ctx, - tag: "dynamic-set", + ctx: ctx, + logger: logger.NOP(), + tag: "dynamic-set", options: option.RuleSet{ Format: C.RuleSetFormatSource, },