Load rule-sets through mmap on iOS

This commit is contained in:
世界 committed 2026-11-09 15:17:20 +00:00
1 parent ada39c1af5
commit d95a2dd87b
16 files changed
+893 -103

No files matched your search

+149
View File
@@ -0,0 +1,149 @@
package ipset
import (
"encoding/binary"
"net/netip"
"slices"
E "github.com/sagernet/sing/common/exceptions"
"go4.org/netipx"
)
type Range4 struct {
From uint32
To uint32
}
type Range6 struct {
From [16]byte
To [16]byte
}
type Set struct {
ranges4 []Range4
ranges6 []Range6
storage any
}
func FromIPSet(set *netipx.IPSet) *Set {
result := &Set{}
for _, ipRange := range set.Ranges() {
from := ipRange.From()
to := ipRange.To()
if from.Is4() {
result.ranges4 = append(result.ranges4, Range4{
From: binary.BigEndian.Uint32(from.AsSlice()),
To: binary.BigEndian.Uint32(to.AsSlice()),
})
} else {
result.ranges6 = append(result.ranges6, Range6{
From: from.As16(),
To: to.As16(),
})
}
}
return result
}
func FromRanges(ranges4 []Range4, ranges6 []Range6, storage any) (*Set, error) {
for i, ipRange := range ranges4 {
if ipRange.From > ipRange.To {
return nil, E.New("ipset: malformed range ", i)
}
if i > 0 && ranges4[i-1].To >= ipRange.From {
return nil, E.New("ipset: unordered range ", i)
}
}
for i, ipRange := range ranges6 {
if compare16(ipRange.From, ipRange.To) > 0 {
return nil, E.New("ipset: malformed range ", i)
}
if i > 0 && compare16(ranges6[i-1].To, ipRange.From) >= 0 {
return nil, E.New("ipset: unordered range ", i)
}
}
return &Set{
ranges4: ranges4,
ranges6: ranges6,
storage: storage,
}, nil
}
func (s *Set) Ranges4() []Range4 {
return s.ranges4
}
func (s *Set) Ranges6() []Range6 {
return s.ranges6
}
func (s *Set) Contains(addr netip.Addr) bool {
if addr.Is4() {
value := binary.BigEndian.Uint32(addr.AsSlice())
index, found := slices.BinarySearchFunc(s.ranges4, value, func(ipRange Range4, target uint32) int {
if ipRange.From > target {
return 1
}
if ipRange.To < target {
return -1
}
return 0
})
_ = index
return found
}
value := addr.As16()
_, found := slices.BinarySearchFunc(s.ranges6, value, func(ipRange Range6, target [16]byte) int {
if compare16(ipRange.From, target) > 0 {
return 1
}
if compare16(ipRange.To, target) < 0 {
return -1
}
return 0
})
return found
}
func (s *Set) IPSet() *netipx.IPSet {
var builder netipx.IPSetBuilder
for _, ipRange := range s.ranges4 {
var from, to [4]byte
binary.BigEndian.PutUint32(from[:], ipRange.From)
binary.BigEndian.PutUint32(to[:], ipRange.To)
builder.AddRange(netipx.IPRangeFrom(netip.AddrFrom4(from), netip.AddrFrom4(to)))
}
for _, ipRange := range s.ranges6 {
builder.AddRange(netipx.IPRangeFrom(netip.AddrFrom16(ipRange.From), netip.AddrFrom16(ipRange.To)))
}
set, err := builder.IPSet()
if err != nil {
panic(err)
}
return set
}
func (s *Set) Prefixes() []netip.Prefix {
return s.IPSet().Prefixes()
}
func compare16(a [16]byte, b [16]byte) int {
aHigh := binary.BigEndian.Uint64(a[:8])
bHigh := binary.BigEndian.Uint64(b[:8])
if aHigh != bHigh {
if aHigh < bHigh {
return -1
}
return 1
}
aLow := binary.BigEndian.Uint64(a[8:])
bLow := binary.BigEndian.Uint64(b[8:])
if aLow != bLow {
if aLow < bLow {
return -1
}
return 1
}
return 0
}
+83 -43
View File
@@ -8,6 +8,7 @@ import (
"net/netip"
"unsafe"
"github.com/sagernet/sing-box/common/ipset"
C "github.com/sagernet/sing-box/constant"
"github.com/sagernet/sing-box/option"
"github.com/sagernet/sing/common"
@@ -80,7 +81,7 @@ func Read(reader io.Reader, recover bool) (ruleSetCompat option.PlainRuleSetComp
ruleSetCompat.Version = version
for i := range length {
var rule option.HeadlessRule
rule, err = readRule(bReader, recover, 0)
rule, err = readRule(bReader, recover, 0, nil)
if err != nil {
err = E.Cause(err, "read rule[", i, "]")
return
@@ -109,7 +110,7 @@ func Write(writer io.Writer, ruleSet option.PlainRuleSet, generateVersion uint8)
return err
}
for _, rule := range ruleSet.Rules {
err = writeRule(bWriter, rule, generateVersion)
err = writeRule(bWriter, rule, generateVersion, nil)
if err != nil {
return err
}
@@ -123,7 +124,7 @@ func Write(writer io.Writer, ruleSet option.PlainRuleSet, generateVersion uint8)
const maxLogicalRuleDepth = 100
func readRule(reader varbin.Reader, recover bool, depth int) (rule option.HeadlessRule, err error) {
func readRule(reader varbin.Reader, recover bool, depth int, mmap *mmapReader) (rule option.HeadlessRule, err error) {
if depth > maxLogicalRuleDepth {
err = E.New("logical rule nested too deep")
return
@@ -136,28 +137,28 @@ func readRule(reader varbin.Reader, recover bool, depth int) (rule option.Headle
switch ruleType {
case 0:
rule.Type = C.RuleTypeDefault
rule.DefaultOptions, err = readDefaultRule(reader, recover)
rule.DefaultOptions, err = readDefaultRule(reader, recover, mmap)
case 1:
rule.Type = C.RuleTypeLogical
rule.LogicalOptions, err = readLogicalRule(reader, recover, depth)
rule.LogicalOptions, err = readLogicalRule(reader, recover, depth, mmap)
default:
err = E.New("unknown rule type: ", ruleType)
}
return
}
func writeRule(writer varbin.Writer, rule option.HeadlessRule, generateVersion uint8) error {
func writeRule(writer varbin.Writer, rule option.HeadlessRule, generateVersion uint8, mmap *mmapWriter) error {
switch rule.Type {
case C.RuleTypeDefault:
return writeDefaultRule(writer, rule.DefaultOptions, generateVersion)
return writeDefaultRule(writer, rule.DefaultOptions, generateVersion, mmap)
case C.RuleTypeLogical:
return writeLogicalRule(writer, rule.LogicalOptions, generateVersion)
return writeLogicalRule(writer, rule.LogicalOptions, generateVersion, mmap)
default:
panic("unknown rule type: " + rule.Type)
}
}
func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHeadlessRule, err error) {
func readDefaultRule(reader varbin.Reader, recover bool, mmap *mmapReader) (rule option.DefaultHeadlessRule, err error) {
var lastItemType uint8
for {
var itemType uint8
@@ -179,7 +180,11 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea
rule.Network, err = readRuleItemString(reader)
case ruleItemDomain:
var matcher *domain.Matcher
matcher, err = domain.ReadMatcher(reader)
if mmap != nil {
matcher, err = mmap.readMatcher(reader)
} else {
matcher, err = domain.ReadMatcher(reader)
}
if err != nil {
return
}
@@ -192,7 +197,11 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea
case ruleItemDomainRegex:
rule.DomainRegex, err = readRuleItemString(reader)
case ruleItemSourceIPCIDR:
rule.SourceIPSet, err = readIPSet(reader)
if mmap != nil {
rule.SourceIPSet, err = mmap.readIPSet(reader)
} else {
rule.SourceIPSet, err = readIPSet(reader)
}
if err != nil {
return
}
@@ -200,7 +209,11 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea
rule.SourceIPCIDR = common.Map(rule.SourceIPSet.Prefixes(), netip.Prefix.String)
}
case ruleItemIPCIDR:
rule.IPSet, err = readIPSet(reader)
if mmap != nil {
rule.IPSet, err = mmap.readIPSet(reader)
} else {
rule.IPSet, err = readIPSet(reader)
}
if err != nil {
return
}
@@ -231,7 +244,11 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea
rule.WIFIBSSID, err = readRuleItemString(reader)
case ruleItemAdGuardDomain:
var matcher *domain.AdGuardMatcher
matcher, err = domain.ReadAdGuardMatcher(reader)
if mmap != nil {
matcher, err = mmap.readAdGuardMatcher(reader)
} else {
matcher, err = domain.ReadAdGuardMatcher(reader)
}
if err != nil {
return
}
@@ -303,7 +320,7 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea
}
}
func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, generateVersion uint8) error {
func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, generateVersion uint8, mmap *mmapWriter) error {
err := binary.Write(writer, binary.BigEndian, uint8(0))
if err != nil {
return err
@@ -322,12 +339,20 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen
return err
}
}
if len(rule.Domain) > 0 || len(rule.DomainSuffix) > 0 {
if len(rule.Domain) > 0 || len(rule.DomainSuffix) > 0 || mmap != nil && rule.DomainMatcher != nil {
err = binary.Write(writer, binary.BigEndian, ruleItemDomain)
if err != nil {
return err
}
err = domain.NewMatcher(rule.Domain, rule.DomainSuffix, generateVersion == C.RuleSetVersion1).Write(writer)
matcher := rule.DomainMatcher
if len(rule.Domain) > 0 || len(rule.DomainSuffix) > 0 {
matcher = domain.NewMatcher(rule.Domain, rule.DomainSuffix, generateVersion == C.RuleSetVersion1)
}
if mmap != nil {
err = mmap.writeMatcher(writer, matcher)
} else {
err = matcher.Write(writer)
}
if err != nil {
return err
}
@@ -344,14 +369,14 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen
return err
}
}
if len(rule.SourceIPCIDR) > 0 {
err = writeRuleItemCIDR(writer, ruleItemSourceIPCIDR, rule.SourceIPCIDR)
if len(rule.SourceIPCIDR) > 0 || mmap != nil && rule.SourceIPSet != nil {
err = writeRuleItemCIDR(writer, ruleItemSourceIPCIDR, rule.SourceIPCIDR, rule.SourceIPSet, mmap)
if err != nil {
return E.Cause(err, "source_ip_cidr")
}
}
if len(rule.IPCIDR) > 0 {
err = writeRuleItemCIDR(writer, ruleItemIPCIDR, rule.IPCIDR)
if len(rule.IPCIDR) > 0 || mmap != nil && rule.IPSet != nil {
err = writeRuleItemCIDR(writer, ruleItemIPCIDR, rule.IPCIDR, rule.IPSet, mmap)
if err != nil {
return E.Cause(err, "ipcidr")
}
@@ -500,7 +525,7 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen
return err
}
}
if len(rule.AdGuardDomain) > 0 {
if len(rule.AdGuardDomain) > 0 || mmap != nil && rule.AdGuardDomainMatcher != nil {
if generateVersion < C.RuleSetVersion2 {
return E.New("AdGuard rule items is only supported in version 2 or later")
}
@@ -508,7 +533,15 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen
if err != nil {
return err
}
err = domain.NewAdGuardMatcher(rule.AdGuardDomain).Write(writer)
matcher := rule.AdGuardDomainMatcher
if len(rule.AdGuardDomain) > 0 {
matcher = domain.NewAdGuardMatcher(rule.AdGuardDomain)
}
if mmap != nil {
err = mmap.writeAdGuardMatcher(writer, matcher)
} else {
err = matcher.Write(writer)
}
if err != nil {
return err
}
@@ -588,33 +621,40 @@ func writeRuleItemUint16(writer varbin.Writer, itemType uint8, value []uint16) e
return binary.Write(writer, binary.BigEndian, value)
}
func writeRuleItemCIDR(writer varbin.Writer, itemType uint8, value []string) error {
var builder netipx.IPSetBuilder
for i, prefixString := range value {
prefix, err := netip.ParsePrefix(prefixString)
if err == nil {
builder.AddPrefix(prefix)
continue
func writeRuleItemCIDR(writer varbin.Writer, itemType uint8, value []string, rawSet *ipset.Set, mmap *mmapWriter) error {
set := rawSet
if len(value) > 0 {
var builder netipx.IPSetBuilder
for i, prefixString := range value {
prefix, err := netip.ParsePrefix(prefixString)
if err == nil {
builder.AddPrefix(prefix)
continue
}
addr, addrErr := netip.ParseAddr(prefixString)
if addrErr == nil {
builder.Add(addr)
continue
}
return E.Cause(err, "parse [", i, "]")
}
addr, addrErr := netip.ParseAddr(prefixString)
if addrErr == nil {
builder.Add(addr)
continue
ipSet, err := builder.IPSet()
if err != nil {
return err
}
return E.Cause(err, "parse [", i, "]")
set = ipset.FromIPSet(ipSet)
}
ipSet, err := builder.IPSet()
err := binary.Write(writer, binary.BigEndian, itemType)
if err != nil {
return err
}
err = binary.Write(writer, binary.BigEndian, itemType)
if err != nil {
return err
if mmap != nil {
return mmap.writeIPSet(writer, set)
}
return writeIPSet(writer, ipSet)
return writeIPSet(writer, set)
}
func readLogicalRule(reader varbin.Reader, recovery bool, depth int) (logicalRule option.LogicalHeadlessRule, err error) {
func readLogicalRule(reader varbin.Reader, recovery bool, depth int, mmap *mmapReader) (logicalRule option.LogicalHeadlessRule, err error) {
mode, err := reader.ReadByte()
if err != nil {
return
@@ -634,7 +674,7 @@ func readLogicalRule(reader varbin.Reader, recovery bool, depth int) (logicalRul
}
for i := range length {
var rule option.HeadlessRule
rule, err = readRule(reader, recovery, depth+1)
rule, err = readRule(reader, recovery, depth+1, mmap)
if err != nil {
err = E.Cause(err, "read logical rule [", i, "]")
return
@@ -648,7 +688,7 @@ func readLogicalRule(reader varbin.Reader, recovery bool, depth int) (logicalRul
return
}
func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRule, generateVersion uint8) error {
func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRule, generateVersion uint8, mmap *mmapWriter) error {
err := binary.Write(writer, binary.BigEndian, uint8(1))
if err != nil {
return err
@@ -669,7 +709,7 @@ func writeLogicalRule(writer varbin.Writer, logicalRule option.LogicalHeadlessRu
return err
}
for _, rule := range logicalRule.Rules {
err = writeRule(writer, rule, generateVersion)
err = writeRule(writer, rule, generateVersion, mmap)
if err != nil {
return err
}
+7 -9
View File
@@ -9,6 +9,7 @@ import (
"testing"
"unsafe"
"github.com/sagernet/sing-box/common/ipset"
M "github.com/sagernet/sing/common/metadata"
"github.com/sagernet/sing/common/varbin"
@@ -94,14 +95,11 @@ func oldReadIPSet(reader varbin.Reader) (*netipx.IPSet, error) {
if err != nil {
return nil, err
}
mySet := &myIPSet{
rr: make([]myIPRange, len(ranges)),
var builder netipx.IPSetBuilder
for _, rangeData := range ranges {
builder.AddRange(netipx.IPRangeFrom(M.AddrFromIP(rangeData.From), M.AddrFromIP(rangeData.To)))
}
for i, rangeData := range ranges {
mySet.rr[i].from = M.AddrFromIP(rangeData.From)
mySet.rr[i].to = M.AddrFromIP(rangeData.To)
}
return (*netipx.IPSet)(unsafe.Pointer(mySet)), nil
return builder.IPSet()
}
// New write functions (without itemType prefix for testing)
@@ -384,7 +382,7 @@ func TestIPSetCompat(t *testing.T) {
// New write
var newBuf bytes.Buffer
err := writeIPSet(&newBuf, tc.input)
err := writeIPSet(&newBuf, ipset.FromIPSet(tc.input))
require.NoError(t, err)
// Verify format starts with version byte (1) + uint64 count
@@ -399,7 +397,7 @@ func TestIPSetCompat(t *testing.T) {
// New write -> new read
readBack2, err := readIPSet(bufio.NewReader(bytes.NewReader(newBuf.Bytes())))
require.NoError(t, err)
requireIPSetEqual(t, tc.input, readBack2)
requireIPSetEqual(t, tc.input, readBack2.IPSet())
})
}
}
+63 -38
View File
@@ -3,26 +3,14 @@ package srs
import (
"encoding/binary"
"io"
"net/netip"
"os"
"unsafe"
"slices"
M "github.com/sagernet/sing/common/metadata"
"github.com/sagernet/sing-box/common/ipset"
"github.com/sagernet/sing/common/varbin"
"go4.org/netipx"
)
type myIPSet struct {
rr []myIPRange
}
type myIPRange struct {
from netip.Addr
to netip.Addr
}
func readIPSet(reader varbin.Reader) (*netipx.IPSet, error) {
func readIPSet(reader varbin.Reader) (*ipset.Set, error) {
version, err := reader.ReadByte()
if err != nil {
return nil, err
@@ -36,67 +24,104 @@ func readIPSet(reader varbin.Reader) (*netipx.IPSet, error) {
if err != nil {
return nil, err
}
mySet := &myIPSet{}
var (
ranges4 []ipset.Range4
ranges6 []ipset.Range6
)
for range length {
var from, to netip.Addr
from, err = readIPSetAddr(reader)
var from, to [16]byte
var fromLength, toLength int
fromLength, err = readIPSetAddr(reader, &from)
if err != nil {
return nil, err
}
to, err = readIPSetAddr(reader)
toLength, err = readIPSetAddr(reader, &to)
if err != nil {
return nil, err
}
mySet.rr = append(mySet.rr, myIPRange{from: from, to: to})
if fromLength != toLength {
return nil, os.ErrInvalid
}
if fromLength == 4 {
ranges4 = append(ranges4, ipset.Range4{
From: binary.BigEndian.Uint32(from[:4]),
To: binary.BigEndian.Uint32(to[:4]),
})
} else {
ranges6 = append(ranges6, ipset.Range6{From: from, To: to})
}
}
return (*netipx.IPSet)(unsafe.Pointer(mySet)), nil
slices.SortFunc(ranges4, func(a, b ipset.Range4) int {
if a.From < b.From {
return -1
} else if a.From > b.From {
return 1
}
return 0
})
slices.SortFunc(ranges6, func(a, b ipset.Range6) int {
return slices.Compare(a.From[:], b.From[:])
})
return ipset.FromRanges(slices.Clip(ranges4), slices.Clip(ranges6), nil)
}
func readIPSetAddr(reader varbin.Reader) (netip.Addr, error) {
func readIPSetAddr(reader varbin.Reader, addr *[16]byte) (int, error) {
addrLen, err := binary.ReadUvarint(reader)
if err != nil {
return netip.Addr{}, err
return 0, err
}
if addrLen != 4 && addrLen != 16 {
return netip.Addr{}, os.ErrInvalid
return 0, os.ErrInvalid
}
var addrBytes [16]byte
_, err = io.ReadFull(reader, addrBytes[:addrLen])
_, err = io.ReadFull(reader, addr[:addrLen])
if err != nil {
return netip.Addr{}, err
return 0, err
}
return M.AddrFromIP(addrBytes[:addrLen]), nil
return int(addrLen), nil
}
func writeIPSet(writer varbin.Writer, set *netipx.IPSet) error {
func writeIPSet(writer varbin.Writer, set *ipset.Set) error {
err := writer.WriteByte(1)
if err != nil {
return err
}
mySet := (*myIPSet)(unsafe.Pointer(set))
err = binary.Write(writer, binary.BigEndian, uint64(len(mySet.rr)))
ranges4 := set.Ranges4()
ranges6 := set.Ranges6()
err = binary.Write(writer, binary.BigEndian, uint64(len(ranges4)+len(ranges6)))
if err != nil {
return err
}
for _, rr := range mySet.rr {
fromBytes := rr.from.AsSlice()
_, err = varbin.WriteUvarint(writer, uint64(len(fromBytes)))
for _, ipRange := range ranges4 {
var from, to [4]byte
binary.BigEndian.PutUint32(from[:], ipRange.From)
binary.BigEndian.PutUint32(to[:], ipRange.To)
err = writeIPSetAddr(writer, from[:])
if err != nil {
return err
}
_, err = writer.Write(fromBytes)
err = writeIPSetAddr(writer, to[:])
if err != nil {
return err
}
toBytes := rr.to.AsSlice()
_, err = varbin.WriteUvarint(writer, uint64(len(toBytes)))
}
for _, ipRange := range ranges6 {
err = writeIPSetAddr(writer, ipRange.From[:])
if err != nil {
return err
}
_, err = writer.Write(toBytes)
err = writeIPSetAddr(writer, ipRange.To[:])
if err != nil {
return err
}
}
return nil
}
func writeIPSetAddr(writer varbin.Writer, addr []byte) error {
_, err := varbin.WriteUvarint(writer, uint64(len(addr)))
if err != nil {
return err
}
_, err = writer.Write(addr)
return err
}
+403
View File
@@ -0,0 +1,403 @@
package srs
import (
"bufio"
"bytes"
"encoding/binary"
"io"
"os"
"runtime"
"unsafe"
"github.com/sagernet/sing-box/common/ipset"
"github.com/sagernet/sing-box/option"
"github.com/sagernet/sing/common/domain"
E "github.com/sagernet/sing/common/exceptions"
"github.com/sagernet/sing/common/varbin"
)
var mmapMagic = [4]byte{0x53, 0x52, 0x53, 0x4D} // SRSM
const (
mmapFormatVersion = 1
mmapHeaderSize = 40
mmapBlobEntrySize = 24
mmapBlobSuccinct = 1
mmapBlobIPSet = 2
)
type mmapStorage struct {
release func()
}
type mmapBlob struct {
kind uint64
matcher domain.Mmap
set *ipset.Set
}
type mmapWriter struct {
blobs []mmapBlob
}
func (w *mmapWriter) writeMatcher(writer varbin.Writer, matcher *domain.Matcher) error {
w.blobs = append(w.blobs, mmapBlob{kind: mmapBlobSuccinct, matcher: matcher.Mmap()})
_, err := varbin.WriteUvarint(writer, uint64(len(w.blobs)-1))
return err
}
func (w *mmapWriter) writeAdGuardMatcher(writer varbin.Writer, matcher *domain.AdGuardMatcher) error {
w.blobs = append(w.blobs, mmapBlob{kind: mmapBlobSuccinct, matcher: matcher.Mmap()})
_, err := varbin.WriteUvarint(writer, uint64(len(w.blobs)-1))
return err
}
func (w *mmapWriter) writeIPSet(writer varbin.Writer, set *ipset.Set) error {
w.blobs = append(w.blobs, mmapBlob{kind: mmapBlobIPSet, set: set})
_, err := varbin.WriteUvarint(writer, uint64(len(w.blobs)-1))
return err
}
func WriteMmap(file io.WriteSeeker, ruleSet option.PlainRuleSetCompat) error {
var header [mmapHeaderSize]byte
_, err := file.Write(header[:])
if err != nil {
return err
}
mmap := &mmapWriter{}
writer := bufio.NewWriter(file)
_, err = varbin.WriteUvarint(writer, uint64(len(ruleSet.Options.Rules)))
if err != nil {
return err
}
for _, rule := range ruleSet.Options.Rules {
err = writeRule(writer, rule, ruleSet.Version, mmap)
if err != nil {
return err
}
}
err = writer.Flush()
if err != nil {
return err
}
rulesEnd, err := file.Seek(0, io.SeekCurrent)
if err != nil {
return err
}
rulesLength := uint64(rulesEnd) - mmapHeaderSize
blobTableOffset := alignMmapOffset(uint64(rulesEnd))
_, err = writer.Write(make([]byte, blobTableOffset-uint64(rulesEnd)))
if err != nil {
return err
}
blobOffset := blobTableOffset + uint64(len(mmap.blobs))*mmapBlobEntrySize
for _, blob := range mmap.blobs {
length := blob.size()
err = writeMmapUint64(writer, blob.kind, blobOffset, length)
if err != nil {
return err
}
blobOffset += length
}
for _, blob := range mmap.blobs {
err = blob.write(writer)
if err != nil {
return err
}
}
err = writer.Flush()
if err != nil {
return err
}
copy(header[0:4], mmapMagic[:])
binary.NativeEndian.PutUint32(header[4:8], mmapFormatVersion)
binary.NativeEndian.PutUint64(header[8:16], uint64(ruleSet.Version))
binary.NativeEndian.PutUint64(header[16:24], rulesLength)
binary.NativeEndian.PutUint64(header[24:32], blobTableOffset)
binary.NativeEndian.PutUint64(header[32:40], uint64(len(mmap.blobs)))
_, err = file.Seek(0, io.SeekStart)
if err != nil {
return err
}
_, err = file.Write(header[:])
return err
}
func (b *mmapBlob) size() uint64 {
switch b.kind {
case mmapBlobSuccinct:
return 40 +
uint64(len(b.matcher.Leaves))*8 +
uint64(len(b.matcher.LabelBitmap))*8 +
alignMmapOffset(uint64(len(b.matcher.Labels))) +
alignMmapOffset(uint64(len(b.matcher.Ranks))*4) +
alignMmapOffset(uint64(len(b.matcher.Selects))*4)
case mmapBlobIPSet:
return 16 +
uint64(len(b.set.Ranges4()))*8 +
uint64(len(b.set.Ranges6()))*32
default:
panic("unknown mmap blob kind")
}
}
func (b *mmapBlob) write(writer *bufio.Writer) error {
switch b.kind {
case mmapBlobSuccinct:
err := writeMmapUint64(writer,
uint64(len(b.matcher.Leaves)),
uint64(len(b.matcher.LabelBitmap)),
uint64(len(b.matcher.Labels)),
uint64(len(b.matcher.Ranks)),
uint64(len(b.matcher.Selects)),
)
if err != nil {
return err
}
err = writeMmapArray(writer, b.matcher.Leaves)
if err != nil {
return err
}
err = writeMmapArray(writer, b.matcher.LabelBitmap)
if err != nil {
return err
}
err = writeMmapArray(writer, b.matcher.Labels)
if err != nil {
return err
}
err = writeMmapArray(writer, b.matcher.Ranks)
if err != nil {
return err
}
return writeMmapArray(writer, b.matcher.Selects)
case mmapBlobIPSet:
err := writeMmapUint64(writer, uint64(len(b.set.Ranges4())), uint64(len(b.set.Ranges6())))
if err != nil {
return err
}
err = writeMmapArray(writer, b.set.Ranges4())
if err != nil {
return err
}
return writeMmapArray(writer, b.set.Ranges6())
default:
panic("unknown mmap blob kind")
}
}
func writeMmapUint64(writer *bufio.Writer, values ...uint64) error {
var buffer [8]byte
for _, value := range values {
binary.NativeEndian.PutUint64(buffer[:], value)
_, err := writer.Write(buffer[:])
if err != nil {
return err
}
}
return nil
}
func writeMmapArray[T any](writer *bufio.Writer, values []T) error {
if len(values) == 0 {
return nil
}
length := uintptr(len(values)) * unsafe.Sizeof(values[0])
_, err := writer.Write(unsafe.Slice((*byte)(unsafe.Pointer(&values[0])), length))
if err != nil {
return err
}
padding := alignMmapOffset(uint64(length)) - uint64(length)
if padding > 0 {
var zero [8]byte
_, err = writer.Write(zero[:padding])
}
return err
}
func alignMmapOffset(offset uint64) uint64 {
return (offset + 7) &^ 7
}
type mmapReader struct {
data []byte
storage *mmapStorage
blobTable []byte
blobCount uint64
}
func ReadMmap(file *os.File) (ruleSet option.PlainRuleSetCompat, err error) {
stat, err := file.Stat()
if err != nil {
return
}
if stat.Size() < mmapHeaderSize {
err = E.New("truncated rule-set mmap")
return
}
data, release, err := mmapFile(file, int(stat.Size()))
if err != nil {
return
}
storage := &mmapStorage{release: release}
cleanup := runtime.AddCleanup(storage, func(release func()) { release() }, release)
defer func() {
if err != nil {
cleanup.Stop()
release()
}
}()
if [4]byte(data[0:4]) != mmapMagic {
err = E.New("invalid rule-set mmap")
return
}
if binary.NativeEndian.Uint32(data[4:8]) != mmapFormatVersion {
err = E.New("unsupported rule-set mmap version")
return
}
rulesLength := binary.NativeEndian.Uint64(data[16:24])
blobTableOffset := binary.NativeEndian.Uint64(data[24:32])
blobCount := binary.NativeEndian.Uint64(data[32:40])
rules, err := mmapSection(data, mmapHeaderSize, rulesLength)
if err != nil {
return
}
blobTable, err := mmapSection(data, blobTableOffset, blobCount*mmapBlobEntrySize)
if err != nil {
return
}
mmap := &mmapReader{
data: data,
storage: storage,
blobTable: blobTable,
blobCount: blobCount,
}
reader := bytes.NewReader(rules)
ruleSet.Version = uint8(binary.NativeEndian.Uint64(data[8:16]))
length, err := binary.ReadUvarint(reader)
if err != nil {
return
}
for i := range length {
var rule option.HeadlessRule
rule, err = readRule(reader, false, 0, mmap)
if err != nil {
err = E.Cause(err, "read rule[", i, "]")
return
}
ruleSet.Options.Rules = append(ruleSet.Options.Rules, rule)
}
return
}
func mmapSection(data []byte, offset uint64, length uint64) ([]byte, error) {
if offset%8 != 0 || offset > uint64(len(data)) || length > uint64(len(data))-offset {
return nil, E.New("rule-set mmap section out of bounds")
}
return data[offset : offset+length : offset+length], nil
}
func (r *mmapReader) blob(reader varbin.Reader, kind uint64) ([]byte, error) {
index, err := binary.ReadUvarint(reader)
if err != nil {
return nil, err
}
if index >= r.blobCount {
return nil, E.New("rule-set mmap blob index out of range")
}
entry := r.blobTable[index*mmapBlobEntrySize:]
if binary.NativeEndian.Uint64(entry[0:8]) != kind {
return nil, E.New("rule-set mmap blob kind mismatch")
}
return mmapSection(r.data, binary.NativeEndian.Uint64(entry[8:16]), binary.NativeEndian.Uint64(entry[16:24]))
}
func (r *mmapReader) readMatcher(reader varbin.Reader) (*domain.Matcher, error) {
mmap, err := r.readSuccinctMmap(reader)
if err != nil {
return nil, err
}
return domain.NewMatcherFromMmap(mmap)
}
func (r *mmapReader) readAdGuardMatcher(reader varbin.Reader) (*domain.AdGuardMatcher, error) {
mmap, err := r.readSuccinctMmap(reader)
if err != nil {
return nil, err
}
return domain.NewAdGuardMatcherFromMmap(mmap)
}
func (r *mmapReader) readSuccinctMmap(reader varbin.Reader) (mmap domain.Mmap, err error) {
blob, err := r.blob(reader, mmapBlobSuccinct)
if err != nil {
return
}
if len(blob) < 40 {
err = E.New("rule-set mmap blob truncated")
return
}
var counts [5]uint64
for i := range counts {
counts[i] = binary.NativeEndian.Uint64(blob[i*8:])
}
cursor := blob[40:]
mmap.Leaves, cursor, err = mmapArray[uint64](cursor, counts[0])
if err != nil {
return
}
mmap.LabelBitmap, cursor, err = mmapArray[uint64](cursor, counts[1])
if err != nil {
return
}
mmap.Labels, cursor, err = mmapArray[byte](cursor, counts[2])
if err != nil {
return
}
mmap.Ranks, cursor, err = mmapArray[int32](cursor, counts[3])
if err != nil {
return
}
mmap.Selects, _, err = mmapArray[int32](cursor, counts[4])
if err != nil {
return
}
mmap.Storage = r.storage
return
}
func (r *mmapReader) readIPSet(reader varbin.Reader) (*ipset.Set, error) {
blob, err := r.blob(reader, mmapBlobIPSet)
if err != nil {
return nil, err
}
if len(blob) < 16 {
return nil, E.New("rule-set mmap blob truncated")
}
cursor := blob[16:]
ranges4, cursor, err := mmapArray[ipset.Range4](cursor, binary.NativeEndian.Uint64(blob[0:8]))
if err != nil {
return nil, err
}
ranges6, _, err := mmapArray[ipset.Range6](cursor, binary.NativeEndian.Uint64(blob[8:16]))
if err != nil {
return nil, err
}
return ipset.FromRanges(ranges4, ranges6, r.storage)
}
func mmapArray[T any](data []byte, count uint64) ([]T, []byte, error) {
var zero T
size := uint64(unsafe.Sizeof(zero))
if count > uint64(len(data))/size {
return nil, nil, E.New("rule-set mmap array out of bounds")
}
length := alignMmapOffset(count * size)
if length > uint64(len(data)) {
return nil, nil, E.New("rule-set mmap array out of bounds")
}
if count == 0 {
return nil, data[length:], nil
}
return unsafe.Slice((*T)(unsafe.Pointer(&data[0])), count), data[length:], nil
}
+13
View File
@@ -0,0 +1,13 @@
//go:build !unix
package srs
import (
"os"
E "github.com/sagernet/sing/common/exceptions"
)
func mmapFile(file *os.File, size int) ([]byte, func(), error) {
return nil, nil, E.New("rule-set mmap is not supported on this platform")
}
+80
View File
@@ -0,0 +1,80 @@
//go:build unix
package srs
import (
"bytes"
"io"
"net/netip"
"os"
"path/filepath"
"testing"
C "github.com/sagernet/sing-box/constant"
"github.com/sagernet/sing-box/option"
"github.com/sagernet/sing/common/json/badoption"
"github.com/stretchr/testify/require"
)
func TestMmapMatchesSource(t *testing.T) {
t.Parallel()
source := option.PlainRuleSet{Rules: []option.HeadlessRule{
{
Type: C.RuleTypeDefault,
DefaultOptions: option.DefaultHeadlessRule{
Domain: badoption.Listable[string]{"example.com"},
DomainSuffix: badoption.Listable[string]{".example.org"},
IPCIDR: badoption.Listable[string]{"10.0.0.0/8", "2001:db8::/32", "192.0.2.1"},
},
},
{
Type: C.RuleTypeLogical,
LogicalOptions: option.LogicalHeadlessRule{
Mode: C.LogicalTypeOr,
Rules: []option.HeadlessRule{{
Type: C.RuleTypeDefault,
DefaultOptions: option.DefaultHeadlessRule{
AdGuardDomain: badoption.Listable[string]{"||ads.example.net^"},
SourceIPCIDR: badoption.Listable[string]{"172.16.0.0/12"},
},
}},
},
},
}}
var binary bytes.Buffer
require.NoError(t, Write(&binary, source, C.RuleSetVersionCurrent))
parsed, err := Read(bytes.NewReader(binary.Bytes()), false)
require.NoError(t, err)
mmapPath := filepath.Join(t.TempDir(), "rule-set.mmap")
mmapFile, err := os.Create(mmapPath)
require.NoError(t, err)
defer mmapFile.Close()
require.NoError(t, WriteMmap(mmapFile, parsed))
_, err = mmapFile.Seek(0, io.SeekStart)
require.NoError(t, err)
mmap, err := ReadMmap(mmapFile)
require.NoError(t, err)
_, err = os.Stat(mmapPath)
require.ErrorIs(t, err, os.ErrNotExist)
require.Equal(t, parsed.Version, mmap.Version)
require.Len(t, mmap.Options.Rules, 2)
first := mmap.Options.Rules[0].DefaultOptions
require.True(t, first.DomainMatcher.Match("example.com"))
require.True(t, first.DomainMatcher.Match("www.example.org"))
require.False(t, first.DomainMatcher.Match("example.org"))
require.False(t, first.DomainMatcher.Match("example.net"))
require.True(t, first.IPSet.Contains(netip.MustParseAddr("10.255.0.1")))
require.True(t, first.IPSet.Contains(netip.MustParseAddr("192.0.2.1")))
require.False(t, first.IPSet.Contains(netip.MustParseAddr("192.0.2.2")))
require.True(t, first.IPSet.Contains(netip.MustParseAddr("2001:db8::1")))
require.False(t, first.IPSet.Contains(netip.MustParseAddr("2001:db9::1")))
second := mmap.Options.Rules[1].LogicalOptions.Rules[0].DefaultOptions
require.True(t, second.AdGuardDomainMatcher.Match("ads.example.net"))
require.False(t, second.AdGuardDomainMatcher.Match("example.net"))
require.True(t, second.SourceIPSet.Contains(netip.MustParseAddr("172.31.255.255")))
require.False(t, second.SourceIPSet.Contains(netip.MustParseAddr("172.32.0.0")))
}
+20
View File
@@ -0,0 +1,20 @@
//go:build unix
package srs
import (
"os"
"golang.org/x/sys/unix"
)
func mmapFile(file *os.File, size int) ([]byte, func(), error) {
data, err := unix.Mmap(int(file.Fd()), 0, size, unix.PROT_READ, unix.MAP_PRIVATE)
if err != nil {
return nil, nil, err
}
os.Remove(file.Name())
return data, func() {
unix.Munmap(data)
}, nil
}
+3 -4
View File
@@ -6,6 +6,7 @@ import (
"reflect"
"strings"
"github.com/sagernet/sing-box/common/ipset"
C "github.com/sagernet/sing-box/constant"
"github.com/sagernet/sing-box/schema"
"github.com/sagernet/sing/common"
@@ -15,8 +16,6 @@ import (
"github.com/sagernet/sing/common/json"
"github.com/sagernet/sing/common/json/badjson"
"github.com/sagernet/sing/common/json/badoption"
"go4.org/netipx"
)
type _RuleSet struct {
@@ -277,8 +276,8 @@ type DefaultHeadlessRule struct {
Invert bool `json:"invert,omitempty"`
DomainMatcher *domain.Matcher `json:"-"`
SourceIPSet *netipx.IPSet `json:"-"`
IPSet *netipx.IPSet `json:"-"`
SourceIPSet *ipset.Set `json:"-"`
IPSet *ipset.Set `json:"-"`
AdGuardDomain badoption.Listable[string] `json:"-"`
AdGuardDomainMatcher *domain.AdGuardMatcher `json:"-"`
+4 -3
View File
@@ -6,6 +6,7 @@ import (
"strings"
"github.com/sagernet/sing-box/adapter"
"github.com/sagernet/sing-box/common/ipset"
"github.com/sagernet/sing/common"
E "github.com/sagernet/sing/common/exceptions"
@@ -15,7 +16,7 @@ import (
var _ RuleItem = (*IPCIDRItem)(nil)
type IPCIDRItem struct {
ipSet *netipx.IPSet
ipSet *ipset.Set
isSource bool
description string
}
@@ -53,13 +54,13 @@ func NewIPCIDRItem(isSource bool, prefixStrings []string) (*IPCIDRItem, error) {
return nil, err
}
return &IPCIDRItem{
ipSet: ipSet,
ipSet: ipset.FromIPSet(ipSet),
isSource: isSource,
description: description,
}, nil
}
func NewRawIPCIDRItem(isSource bool, ipSet *netipx.IPSet) *IPCIDRItem {
func NewRawIPCIDRItem(isSource bool, ipSet *ipset.Set) *IPCIDRItem {
var description string
if isSource {
description = "source_ip_cidr="
+1 -1
View File
@@ -32,7 +32,7 @@ func extractIPSetFromRule(rawRule adapter.HeadlessRule) []*netipx.IPSet {
return common.FlatMap(rule.destinationIPCIDRItems, func(rawItem RuleItem) []*netipx.IPSet {
switch item := rawItem.(type) {
case *IPCIDRItem:
return []*netipx.IPSet{item.ipSet}
return []*netipx.IPSet{item.ipSet.IPSet()}
default:
return nil
}
+1 -2
View File
@@ -107,7 +107,6 @@ func (s *LocalRuleSet) reloadFile(path string) error {
if err != nil {
return err
}
case C.RuleSetFormatBinary:
setFile, err := filemanager.Open(s.ctx, path)
if err != nil {
@@ -121,7 +120,7 @@ func (s *LocalRuleSet) reloadFile(path string) error {
default:
return E.New("unknown rule-set format: ", s.fileFormat)
}
plainRuleSet, err := ruleSet.Upgrade()
plainRuleSet, err := mmapRuleSet(s.ctx, s.logger, s.tag, ruleSet).Upgrade()
if err != nil {
return err
}
+44
View File
@@ -0,0 +1,44 @@
//go:build ios
package rule
import (
"context"
"io"
"os"
"github.com/sagernet/sing-box/common/srs"
"github.com/sagernet/sing-box/option"
E "github.com/sagernet/sing/common/exceptions"
"github.com/sagernet/sing/common/logger"
"github.com/sagernet/sing/service/filemanager"
)
func mmapRuleSet(ctx context.Context, logger logger.Logger, tag string, ruleSet option.PlainRuleSetCompat) option.PlainRuleSetCompat {
file, err := filemanager.CreateTemp(ctx, "rule-set-*.mmap")
if err != nil {
logger.Warn(E.Cause(err, "create rule-set mmap file ", tag))
return ruleSet
}
defer file.Close()
mmap, err := writeAndMmapRuleSet(file, ruleSet)
if err != nil {
filemanager.Remove(ctx, file.Name())
logger.Warn(E.Cause(err, "mmap rule-set ", tag))
return ruleSet
}
logger.Debug("mmap rule-set ", tag)
return mmap
}
func writeAndMmapRuleSet(file *os.File, ruleSet option.PlainRuleSetCompat) (option.PlainRuleSetCompat, error) {
err := srs.WriteMmap(file, ruleSet)
if err != nil {
return ruleSet, err
}
_, err = file.Seek(0, io.SeekStart)
if err != nil {
return ruleSet, err
}
return srs.ReadMmap(file)
}
+14
View File
@@ -0,0 +1,14 @@
//go:build !ios
package rule
import (
"context"
"github.com/sagernet/sing-box/option"
"github.com/sagernet/sing/common/logger"
)
func mmapRuleSet(ctx context.Context, logger logger.Logger, tag string, ruleSet option.PlainRuleSetCompat) option.PlainRuleSetCompat {
return ruleSet
}
+1 -1
View File
@@ -197,7 +197,7 @@ func (s *RemoteRuleSet) loadBytes(content []byte) error {
default:
return E.New("unknown rule-set format: ", s.options.Format)
}
plainRuleSet, err := ruleSet.Upgrade()
plainRuleSet, err := mmapRuleSet(s.ctx, s.logger, s.tag, ruleSet).Upgrade()
if err != nil {
return err
}
@@ -2,6 +2,7 @@ package rule
import (
"context"
"os"
"sync/atomic"
"testing"
@@ -10,8 +11,10 @@ import (
"github.com/sagernet/sing-box/option"
E "github.com/sagernet/sing/common/exceptions"
"github.com/sagernet/sing/common/json/badoption"
"github.com/sagernet/sing/common/logger"
"github.com/sagernet/sing/common/x/list"
"github.com/sagernet/sing/service"
"github.com/sagernet/sing/service/filemanager"
"github.com/stretchr/testify/require"
)
@@ -85,9 +88,11 @@ func TestRemoteRuleSetLoadBytesRejectsInvalidUpdateBeforeCommit(t *testing.T) {
return nil
},
})
ctx = filemanager.WithDefault(ctx, "", t.TempDir(), os.Getuid(), os.Getgid())
ruleSet := &RemoteRuleSet{
ctx: ctx,
tag: "dynamic-set",
ctx: ctx,
logger: logger.NOP(),
tag: "dynamic-set",
options: option.RuleSet{
Format: C.RuleSetFormatSource,
},