Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ linters-settings:

gosec:
excludes:
- G404 # 随机数生成器可能不是密码学安全的
- G404 # Random number generator may not be cryptographically secure

gocritic:
enabled-tags:
Expand All @@ -50,30 +50,30 @@ linters-settings:

issues:
exclude-rules:
# 排除测试文件中的某些检查
# Exclude selected checks in test files
- path: _test\.go
linters:
- gosec
- errcheck
- gocyclo

# 排除CASE表达式测试中的复杂度检查
# Exclude complexity checks in CASE expression tests
- path: streamsql_case_test\.go
linters:
- gocyclo
- funlen

# 排除generated文件
# Exclude generated files
- path: ".*\\.pb\\.go"
linters:
- all

exclude:
# 排除一些常见的false positive
# Exclude common false positives
- "Error return value of .((os\\.)?std(out|err)\\..*|.*Close|.*Flush|os\\.Remove(All)?|.*print.*|os\\.(Un)?Setenv). is not checked"
- "should have a package comment"

output:
format: colored-line-number
print-issued-lines: true
print-linter-name: true
print-linter-name: true
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
[![codecov](https://codecov.io/gh/rulego/streamsql/graph/badge.svg?token=1CK1O5J1BI)](https://codecov.io/gh/rulego/streamsql)
[![Mentioned in Awesome Go](https://awesome.re/mentioned-badge.svg)](https://github.com/avelino/awesome-go?tab=readme-ov-file#stream-processing)

English | [简体中文](README_ZH.md)
English | [Chinese Simplified](README_ZH.md)

**StreamSQL** is an embeddable, SQL-based stream processing engine built for the IoT edge. It sits between a **time-series database** and **Apache Flink**: Flink-grade real-time computation with TSDB-grade lightweight deployment — run real-time filtering, windowed aggregation, CDC-style change detection, and complex event pattern matching, in-process, inside a 128MB gateway.

Expand Down
6 changes: 3 additions & 3 deletions aggregator/builtin.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ type AggregatorFunction = functions.LegacyAggregatorFunction
// ContextAggregator aggregator interface supporting context mechanism, re-exports functions.ContextAggregator
type ContextAggregator = functions.ContextAggregator

// Register 将自定义聚合器注册到全局 registryre-export functions.RegisterLegacyAggregator)。
// 仅作 legacy 兜底:自定义聚合请实现 functions.AggregatorFunction 接口并用 functions.Register 注册,
// 适配器会自动接通,无需调用本函数。
// Register: Registers custom aggregators to the global registry (re-export functions.RegisterLegacyAggregator).
// Only as a legacy backup: For custom aggregation, please implement the functions.AggregatorFunction interface and register with functions.Register,
// The adapter will automatically turn on without calling this function.
func Register(name string, constructor func() AggregatorFunction) {
functions.RegisterLegacyAggregator(name, constructor)
}
Expand Down
48 changes: 24 additions & 24 deletions aggregator/builtin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,67 +7,67 @@ import (
"github.com/stretchr/testify/require"
)

// TestPostAggregationPlaceholder 测试后聚合占位符的完整功能
// TestPostAggregationPlaceholder Tests the full functionality of aggregated placeholders
func TestPostAggregationPlaceholder(t *testing.T) {
t.Run("测试PostAggregationPlaceholder基本功能", func(t *testing.T) {
// 创建PostAggregationPlaceholder实例
// Create a PostAggregationPlaceholder instance
placeholder := &PostAggregationPlaceholder{}
require.NotNil(t, placeholder)

// 测试New方法
// Test the new method
newPlaceholder := placeholder.New()
require.NotNil(t, newPlaceholder)
assert.IsType(t, &PostAggregationPlaceholder{}, newPlaceholder)

// 测试Add方法(应该不做任何操作)
// Test the Add method (no action should be done)
placeholder.Add(10)
placeholder.Add("test")
placeholder.Add(nil)
placeholder.Add([]int{1, 2, 3})

// 测试Result方法(应该返回nil)
// Test the Result method (should return nil)
result := placeholder.Result()
assert.Nil(t, result)
})

t.Run("测试通过CreateBuiltinAggregator创建PostAggregationPlaceholder", func(t *testing.T) {
// 使用CreateBuiltinAggregator创建post_aggregation类型的聚合器
// Use CreateBuiltinAggregator to create aggregators of post_aggregation type
aggregator := CreateBuiltinAggregator(PostAggregation)
require.NotNil(t, aggregator)
assert.IsType(t, &PostAggregationPlaceholder{}, aggregator)

// 测试创建的聚合器功能
// Test the aggregator features you create
newAgg := aggregator.New()
require.NotNil(t, newAgg)
assert.IsType(t, &PostAggregationPlaceholder{}, newAgg)

// 测试添加各种类型的值
// Test adds various types of values
newAgg.Add(100)
newAgg.Add("string_value")
newAgg.Add(map[string]any{"key": "value"})

// 验证结果始终为nil
// The verification result is always nil
result := newAgg.Result()
assert.Nil(t, result)
})

t.Run("测试PostAggregationPlaceholder的多实例独立性", func(t *testing.T) {
// 创建多个实例
// Create multiple instances
placeholder1 := &PostAggregationPlaceholder{}
placeholder2 := placeholder1.New()
placeholder3 := placeholder1.New()

// 验证实例类型正确
// Verify that the instance type is correct
assert.IsType(t, &PostAggregationPlaceholder{}, placeholder1)
assert.IsType(t, &PostAggregationPlaceholder{}, placeholder2)
assert.IsType(t, &PostAggregationPlaceholder{}, placeholder3)

// 每个实例都应该返回nil
// Each instance should return nil
assert.Nil(t, placeholder1.Result())
assert.Nil(t, placeholder2.Result())
assert.Nil(t, placeholder3.Result())

// 验证Add操作不会影响结果(因为是占位符)
// Verifying that the Add operation does not affect the result (since it is a placeholder)
placeholder1.Add("test1")
placeholder2.Add("test2")
placeholder3.Add("test3")
Expand All @@ -77,18 +77,18 @@ func TestPostAggregationPlaceholder(t *testing.T) {
})

t.Run("测试PostAggregationPlaceholder在聚合场景中的使用", func(t *testing.T) {
// 创建包含PostAggregationPlaceholder的聚合字段
// Create an aggregate field containing PostAggregationPlaceholder
groupFields := []string{"category"}
aggFields := []AggregationField{
{InputField: "value", AggregateType: Sum, OutputAlias: "sum_value"},
{InputField: "placeholder_field", AggregateType: PostAggregation, OutputAlias: "post_agg_field"},
}

// 创建分组聚合器
// Create a packet aggregator
agg := NewGroupAggregator(groupFields, aggFields)
require.NotNil(t, agg)

// 添加测试数据
// Add test data
testData := []map[string]any{
{"category": "A", "value": 10, "placeholder_field": "should_be_ignored"},
{"category": "A", "value": 20, "placeholder_field": "also_ignored"},
Expand All @@ -100,23 +100,23 @@ func TestPostAggregationPlaceholder(t *testing.T) {
assert.NoError(t, err)
}

// 获取结果
// Get results
results, err := agg.GetResults()
assert.NoError(t, err)
assert.Len(t, results, 2)

// 验证PostAggregationPlaceholder字段的结果为nil
// The result of verifying the PostAggregationPlaceholder field is nil
for _, result := range results {
assert.Contains(t, result, "post_agg_field")
assert.Nil(t, result["post_agg_field"])
// 验证正常聚合字段工作正常
// Verify that the aggregated fields are working properly
assert.Contains(t, result, "sum_value")
assert.NotNil(t, result["sum_value"])
}
})
}

// TestCreateBuiltinAggregatorPostAggregation 测试CreateBuiltinAggregator对post_aggregation类型的处理
// TestCreateBuiltinAggregatorPostAggregation Test CreateBuiltinAggregator's handling of post_aggregation types
func TestCreateBuiltinAggregatorPostAggregation(t *testing.T) {
t.Run("测试post_aggregation类型聚合器创建", func(t *testing.T) {
aggregator := CreateBuiltinAggregator("post_aggregation")
Expand All @@ -125,22 +125,22 @@ func TestCreateBuiltinAggregatorPostAggregation(t *testing.T) {
})

t.Run("测试PostAggregation常量", func(t *testing.T) {
// 验证PostAggregation常量值
// Verify the PostAggregation constant value
assert.Equal(t, AggregateType("post_aggregation"), PostAggregation)

// 使用常量创建聚合器
// Create aggregators using constants
aggregator := CreateBuiltinAggregator(PostAggregation)
require.NotNil(t, aggregator)
assert.IsType(t, &PostAggregationPlaceholder{}, aggregator)
})

t.Run("测试与其他聚合类型的区别", func(t *testing.T) {
// 创建不同类型的聚合器
// Create different types of aggregators
sumAgg := CreateBuiltinAggregator(Sum)
countAgg := CreateBuiltinAggregator(Count)
postAgg := CreateBuiltinAggregator(PostAggregation)

// 验证类型不同
// Different types of verification
assert.NotEqual(t, sumAgg, postAgg)
assert.NotEqual(t, countAgg, postAgg)
assert.IsType(t, &PostAggregationPlaceholder{}, postAgg)
Expand Down
20 changes: 10 additions & 10 deletions aggregator/group_aggregator.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ import (
// realistic field values.
const nullGroupKeyMarker = "\x00NULL"

// groupKeySep 分隔分组键各字段。\x1f(单元分隔符)在真实数据中极少出现,避免字段值含
// 分隔符导致的键碰撞(曾用 "|":含 "|" 的值会被还原阶段截断、多字段还会错位)。
// groupKeySep separates the fields of the grouping key. \x1f (cell separator) rarely appears in real data to avoid field values containing
// Key collisions caused by delimiters (previously used with "|": Values containing "|" will be truncated during the restore phase, and multiple fields will be misaligned).
const groupKeySep = "\x1f"

// Aggregator aggregator interface
Expand All @@ -43,7 +43,7 @@ type GroupAggregator struct {
groupFields []string
aggregators map[string]AggregatorFunction
groups map[string]map[string]AggregatorFunction
groupKeyVals map[string][]any // 每个 group key 对应的原始类型分组字段值,供 GetResults 还原(避免序列化丢类型)
groupKeyVals map[string][]any // Each group key corresponds to the original type grouping field value, which GetResults restores (avoiding serialized type loss)
mu sync.RWMutex
context map[string]any
// Expression evaluators
Expand Down Expand Up @@ -152,17 +152,17 @@ func (ga *GroupAggregator) isNumericAggregator(aggType AggregateType) bool {
return false
}

// shouldAllowNullValues 判断聚合函数是否应该允许NULL值
// shouldAllowNullValues determines whether the aggregator should allow NULL values
func (ga *GroupAggregator) shouldAllowNullValues(aggType AggregateType) bool {
// FIRST_VALUE和LAST_VALUE函数应该允许NULL值,因为它们需要记录第一个/最后一个值,即使是NULL
// FIRST_VALUE and LAST_VALUE functions should allow NULL values because they need to record the first/last value, even if it is NULL
return aggType == FirstValue || aggType == LastValue
}

func (ga *GroupAggregator) Add(data any) error {
ga.mu.Lock()
defer ga.mu.Unlock()

// 检查数据是否为nil
// Check if the data is nil
if data == nil {
return fmt.Errorf("data cannot be nil")
}
Expand All @@ -178,7 +178,7 @@ func (ga *GroupAggregator) Add(data any) error {
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
// 检查是否为支持的数据类型
// Check if the data type is supported
if v.Kind() != reflect.Struct && v.Kind() != reflect.Map {
return fmt.Errorf("unsupported data type: %T, expected struct or map", data)
}
Expand Down Expand Up @@ -327,7 +327,7 @@ func (ga *GroupAggregator) Add(data any) error {
groupAgg.Add(numVal)
}
} else {
// 非数值跳过该字段,不中断整行 Add
// Non-numeric values skip the field without interrupting the entire line of Add.
continue
}
} else {
Expand All @@ -346,7 +346,7 @@ func (ga *GroupAggregator) GetResults() ([]map[string]any, error) {
ga.mu.RLock()
defer ga.mu.RUnlock()

// 如果既没有分组字段又没有聚合字段,但有数据被添加过,返回一个空的结果行
// If there are no grouping fields or aggregate fields, but data has been added, return an empty result row
if len(ga.aggregationFields) == 0 && len(ga.groupFields) == 0 {
if len(ga.groups) > 0 {
return []map[string]any{{}}, nil
Expand All @@ -360,7 +360,7 @@ func (ga *GroupAggregator) GetResults() ([]map[string]any, error) {
keyVals := ga.groupKeyVals[key]
for i, field := range ga.groupFields {
if i < len(keyVals) {
group[field] = keyVals[i] // NULL 组此处即 nil
group[field] = keyVals[i] // The NULL group here is nil
}
}
for field, agg := range aggregators {
Expand Down
Loading