add some filter function

This commit is contained in:
@s.roertgen 2025-08-26 16:56:19 +02:00
parent 956860676a
commit 5e16129e4c
4 changed files with 212 additions and 30 deletions

8
go.mod
View file

@ -3,7 +3,7 @@ module github.com/edufeed-org/eventstore
go 1.24.1
require (
github.com/fiatjaf/eventstore v0.16.2
github.com/fiatjaf/eventstore v0.16.4
github.com/nbd-wtf/go-nostr v0.51.8
github.com/stretchr/testify v1.10.0
)
@ -12,10 +12,10 @@ require (
github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
github.com/bytedance/sonic v1.13.1 // indirect
github.com/bytedance/sonic v1.13.2 // indirect
github.com/bytedance/sonic/loader v0.2.4 // indirect
github.com/cloudwego/base64x v0.1.5 // indirect
github.com/coder/websocket v1.8.12 // indirect
github.com/coder/websocket v1.8.13 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
@ -33,6 +33,8 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
golang.org/x/arch v0.15.0 // indirect
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/sys v0.31.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

17
go.sum
View file

@ -4,16 +4,14 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurT
github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ=
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
github.com/bytedance/sonic v1.13.1 h1:Jyd5CIvdFnkOWuKXr+wm4Nyk2h0yAFsr8ucJgEasO3g=
github.com/bytedance/sonic v1.13.1/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY=
github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo=
github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -22,8 +20,7 @@ github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPc
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
github.com/fiatjaf/eventstore v0.16.2 h1:h4rHwSwPcqAKqWUsAbYWUhDeSgm2Kp+PBkJc3FgBYu4=
github.com/fiatjaf/eventstore v0.16.2/go.mod h1:0gU8fzYO/bG+NQAVlHtJWOlt3JKKFefh5Xjj2d1dLIs=
github.com/fiatjaf/eventstore v0.16.4 h1:pENYeuhawxMxlJk8HpRy3pb2oap0fwbphzUgsy7QPws=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@ -33,6 +30,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -46,6 +45,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg=
github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@ -70,12 +70,11 @@ golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -17,15 +17,22 @@ func (ts *TSBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan
ch := make(chan *nostr.Event)
log.Printf("Processing query with search: %s", filter.Search)
// If we have no search parameter, return an empty channel
if filter.Search == "" {
log.Printf("No search parameter provided, returning empty result")
close(ch)
return ch, nil
// Determine the limit for results (default to 100 if not specified)
limit := 100
if filter.Limit > 0 {
limit = filter.Limit
}
nostrsearch, err := ts.SearchResources(filter.Search)
// Use empty search string or the provided search string
searchStr := filter.Search
if searchStr == "" {
log.Printf("No search parameter provided, querying all documents with limit %d", limit)
} else {
log.Printf("Processing query with search: %s and limit %d", searchStr, limit)
}
nostrsearch, err := ts.SearchResourcesWithLimit(searchStr, limit)
if err != nil {
log.Printf("Search failed: %v", err)
// Return the channel anyway, but close it immediately
@ -57,7 +64,7 @@ func (ts *TSBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan
close(ch)
}
}()
return ch, nil
}
@ -126,6 +133,76 @@ func (ts *TSBackend) SearchResources(searchStr string) ([]nostr.Event, error) {
return parseSearchResponse(body)
}
// searches for resources with limit support and returns both the AMB metadata and converted Nostr events
func (ts *TSBackend) SearchResourcesWithLimit(searchStr string, limit int) ([]nostr.Event, error) {
parsedQuery := ParseSearchQuery(searchStr)
mainQuery, params, err := BuildTypesenseQuery(parsedQuery)
if err != nil {
return nil, fmt.Errorf("error building Typesense query: %v", err)
}
// If no search terms provided, use wildcard to match all documents
if mainQuery == "" {
mainQuery = "*"
}
// URL encode the main query
encodedQuery := url.QueryEscape(mainQuery)
// Default fields to search in
queryBy := "name,description,about,learningResourceType,keywords,creator,publisher"
// Start building the search URL with limit
searchURL := fmt.Sprintf("%s/collections/%s/documents/search?validate_field_names=false&q=%s&query_by=%s&per_page=%d",
ts.Host, ts.CollectionName, encodedQuery, queryBy, limit)
// Add additional parameters
for key, value := range params {
searchURL += fmt.Sprintf("&%s=%s", key, url.QueryEscape(value))
}
// Debug information
fmt.Printf("Search URL: %s\n", searchURL)
resp, body, err := ts.makehttpRequest(searchURL, http.MethodGet, nil)
if err != nil {
return nil, fmt.Errorf("error reading response body: %v", err)
}
// Check for errors
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("search failed with status code %d: %s", resp.StatusCode, string(body))
}
// Try to parse the raw JSON to understand its structure
var rawResponse interface{}
if err := json.Unmarshal(body, &rawResponse); err != nil {
fmt.Printf("Warning: Could not parse raw response as JSON: %v\n", err)
} else {
// Check if we got a hits array
responseMap, ok := rawResponse.(map[string]interface{})
if ok {
if hits, exists := responseMap["hits"]; exists {
hitsArray, ok := hits.([]interface{})
if ok {
fmt.Printf("Response contains %d hits\n", len(hitsArray))
if len(hitsArray) > 0 {
// Look at the structure of the first hit
firstHit, ok := hitsArray[0].(map[string]interface{})
if ok {
fmt.Printf("First hit keys: %v\n", getMapKeys(firstHit))
}
}
}
}
}
}
return parseSearchResponse(body)
}
// SearchQuery represents a parsed search query with raw terms and field filters
type SearchQuery struct {
RawTerms []string
@ -154,7 +231,7 @@ func ParseSearchQuery(searchStr string) SearchQuery {
// This is a field:value pair with dot notation
fieldName := match[2]
fieldValue := match[3]
// Add to the array of values for this field
query.FieldFilters[fieldName] = append(query.FieldFilters[fieldName], fieldValue)
} else if match[4] != "" {
@ -164,7 +241,7 @@ func ParseSearchQuery(searchStr string) SearchQuery {
// Simple field:value without dot notation
fieldName := parts[0]
fieldValue := parts[1]
// Add to the array of values for this field
query.FieldFilters[fieldName] = append(query.FieldFilters[fieldName], fieldValue)
} else {
@ -198,7 +275,7 @@ func BuildTypesenseQuery(query SearchQuery) (string, map[string]string, error) {
for _, value := range values {
// Create the filter expression
filterExpr := fmt.Sprintf("%s:%s", field, value)
// Add to the corresponding field group
fieldGroups[baseName] = append(fieldGroups[baseName], filterExpr)
}
@ -234,44 +311,44 @@ func parseSearchResponse(responseBody []byte) ([]nostr.Event, error) {
// Debug: Print the raw response structure
fmt.Printf("Search response found %d hits\n", searchResponse.Found)
nostrResults := make([]nostr.Event, 0, len(searchResponse.Hits))
for i, hit := range searchResponse.Hits {
// Debug: Print hit structure information
fmt.Printf("Processing hit %d, keys: %v\n", i, getMapKeys(hit))
// Check if document exists in the hit
docRaw, exists := hit["document"]
if !exists {
fmt.Printf("Warning: hit %d has no 'document' field\n", i)
continue // Skip this hit
}
// Extract document directly as a map[string]interface{}
docMap, ok := docRaw.(map[string]interface{})
if !ok {
fmt.Printf("Warning: hit %d document is not a map, type: %T\n", i, docRaw)
continue // Skip this hit
}
// Debug: Print document keys
fmt.Printf("Document keys: %v\n", getMapKeys(docMap))
// Check for EventRaw field directly
eventRawVal, hasEventRaw := docMap["eventRaw"]
if !hasEventRaw {
fmt.Printf("Warning: document has no 'eventRaw' field\n")
continue // Skip this document
}
// Try to extract EventRaw as string
eventRawStr, ok := eventRawVal.(string)
if !ok {
fmt.Printf("Warning: eventRaw is not a string, type: %T\n", eventRawVal)
continue // Skip this document
}
// Convert the EventRaw string to a Nostr event
nostrEvent, err := StringifiedJSONToNostrEvent(eventRawStr)
if err != nil {

View file

@ -1,2 +1,106 @@
package typesense30142
import (
"testing"
"github.com/nbd-wtf/go-nostr"
)
func TestQueryEventsWithLimit(t *testing.T) {
// Test that limit is properly handled
tests := []struct {
name string
filter nostr.Filter
expectedLimit int
}{
{
name: "No limit specified - should default to 100",
filter: nostr.Filter{
Search: "",
},
expectedLimit: 100,
},
{
name: "Limit of 2 specified",
filter: nostr.Filter{
Search: "",
Limit: 2,
},
expectedLimit: 2,
},
{
name: "Limit of 50 specified",
filter: nostr.Filter{
Search: "test",
Limit: 50,
},
expectedLimit: 50,
},
{
name: "Zero limit - should default to 100",
filter: nostr.Filter{
Search: "test",
Limit: 0,
},
expectedLimit: 100,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Test the limit logic
limit := 100
if tt.filter.Limit > 0 {
limit = tt.filter.Limit
}
if limit != tt.expectedLimit {
t.Errorf("Expected limit %d, got %d", tt.expectedLimit, limit)
}
})
}
}
func TestBuildTypesenseQueryWithEmptySearch(t *testing.T) {
// Test that empty search strings are handled correctly
query := ParseSearchQuery("")
mainQuery, params, err := BuildTypesenseQuery(query)
if err != nil {
t.Errorf("BuildTypesenseQuery failed: %v", err)
}
// Empty search should result in empty main query
if mainQuery != "" {
t.Errorf("Expected empty main query, got: %s", mainQuery)
}
// Should have no filter parameters for empty search
if len(params) != 0 {
t.Errorf("Expected no filter params for empty search, got: %v", params)
}
}
func TestSearchResourcesWithLimitHandlesEmptySearch(t *testing.T) {
// Test that SearchResourcesWithLimit properly handles empty search by using wildcard
query := ParseSearchQuery("")
mainQuery, _, err := BuildTypesenseQuery(query)
if err != nil {
t.Errorf("BuildTypesenseQuery failed: %v", err)
}
// Empty search should result in empty main query initially
if mainQuery != "" {
t.Errorf("Expected empty main query from ParseSearchQuery, got: %s", mainQuery)
}
// SearchResourcesWithLimit should convert empty query to "*"
if mainQuery == "" {
mainQuery = "*"
}
if mainQuery != "*" {
t.Errorf("Expected wildcard query '*', got: %s", mainQuery)
}
}