diff --git a/go.mod b/go.mod index cc6b795..d9b8a29 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/edufeed-org/eventstore go 1.24.1 require ( - github.com/fiatjaf/eventstore v0.16.2 + github.com/fiatjaf/eventstore v0.16.4 github.com/nbd-wtf/go-nostr v0.51.8 github.com/stretchr/testify v1.10.0 ) @@ -12,10 +12,10 @@ require ( github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect - github.com/bytedance/sonic v1.13.1 // indirect + github.com/bytedance/sonic v1.13.2 // indirect github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cloudwego/base64x v0.1.5 // indirect - github.com/coder/websocket v1.8.12 // indirect + github.com/coder/websocket v1.8.13 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect @@ -33,6 +33,8 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect golang.org/x/arch v0.15.0 // indirect golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect + golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.31.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index bd6d4ec..0c48da3 100644 --- a/go.sum +++ b/go.sum @@ -4,16 +4,14 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurT github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/bytedance/sonic v1.13.1 h1:Jyd5CIvdFnkOWuKXr+wm4Nyk2h0yAFsr8ucJgEasO3g= -github.com/bytedance/sonic v1.13.1/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= -github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= +github.com/coder/websocket v1.8.13 h1:f3QZdXy7uGVz+4uCJy2nTZyM0yTBj8yANEHhqlXZ9FE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -22,8 +20,7 @@ github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/fiatjaf/eventstore v0.16.2 h1:h4rHwSwPcqAKqWUsAbYWUhDeSgm2Kp+PBkJc3FgBYu4= -github.com/fiatjaf/eventstore v0.16.2/go.mod h1:0gU8fzYO/bG+NQAVlHtJWOlt3JKKFefh5Xjj2d1dLIs= +github.com/fiatjaf/eventstore v0.16.4 h1:pENYeuhawxMxlJk8HpRy3pb2oap0fwbphzUgsy7QPws= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -33,6 +30,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -46,6 +45,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg= github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -70,12 +70,11 @@ golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/typesense30142/query.go b/typesense30142/query.go index 643400a..194321f 100644 --- a/typesense30142/query.go +++ b/typesense30142/query.go @@ -17,15 +17,22 @@ func (ts *TSBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan ch := make(chan *nostr.Event) log.Printf("Processing query with search: %s", filter.Search) - - // If we have no search parameter, return an empty channel - if filter.Search == "" { - log.Printf("No search parameter provided, returning empty result") - close(ch) - return ch, nil + + // Determine the limit for results (default to 100 if not specified) + limit := 100 + if filter.Limit > 0 { + limit = filter.Limit } - nostrsearch, err := ts.SearchResources(filter.Search) + // Use empty search string or the provided search string + searchStr := filter.Search + if searchStr == "" { + log.Printf("No search parameter provided, querying all documents with limit %d", limit) + } else { + log.Printf("Processing query with search: %s and limit %d", searchStr, limit) + } + + nostrsearch, err := ts.SearchResourcesWithLimit(searchStr, limit) if err != nil { log.Printf("Search failed: %v", err) // Return the channel anyway, but close it immediately @@ -57,7 +64,7 @@ func (ts *TSBackend) QueryEvents(ctx context.Context, filter nostr.Filter) (chan close(ch) } }() - + return ch, nil } @@ -126,6 +133,76 @@ func (ts *TSBackend) SearchResources(searchStr string) ([]nostr.Event, error) { return parseSearchResponse(body) } +// searches for resources with limit support and returns both the AMB metadata and converted Nostr events +func (ts *TSBackend) SearchResourcesWithLimit(searchStr string, limit int) ([]nostr.Event, error) { + parsedQuery := ParseSearchQuery(searchStr) + + mainQuery, params, err := BuildTypesenseQuery(parsedQuery) + if err != nil { + return nil, fmt.Errorf("error building Typesense query: %v", err) + } + + // If no search terms provided, use wildcard to match all documents + if mainQuery == "" { + mainQuery = "*" + } + + // URL encode the main query + encodedQuery := url.QueryEscape(mainQuery) + + // Default fields to search in + queryBy := "name,description,about,learningResourceType,keywords,creator,publisher" + + // Start building the search URL with limit + searchURL := fmt.Sprintf("%s/collections/%s/documents/search?validate_field_names=false&q=%s&query_by=%s&per_page=%d", + ts.Host, ts.CollectionName, encodedQuery, queryBy, limit) + + // Add additional parameters + for key, value := range params { + searchURL += fmt.Sprintf("&%s=%s", key, url.QueryEscape(value)) + } + + // Debug information + fmt.Printf("Search URL: %s\n", searchURL) + + resp, body, err := ts.makehttpRequest(searchURL, http.MethodGet, nil) + + if err != nil { + return nil, fmt.Errorf("error reading response body: %v", err) + } + + // Check for errors + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("search failed with status code %d: %s", resp.StatusCode, string(body)) + } + + // Try to parse the raw JSON to understand its structure + var rawResponse interface{} + if err := json.Unmarshal(body, &rawResponse); err != nil { + fmt.Printf("Warning: Could not parse raw response as JSON: %v\n", err) + } else { + // Check if we got a hits array + responseMap, ok := rawResponse.(map[string]interface{}) + if ok { + if hits, exists := responseMap["hits"]; exists { + hitsArray, ok := hits.([]interface{}) + if ok { + fmt.Printf("Response contains %d hits\n", len(hitsArray)) + if len(hitsArray) > 0 { + // Look at the structure of the first hit + firstHit, ok := hitsArray[0].(map[string]interface{}) + if ok { + fmt.Printf("First hit keys: %v\n", getMapKeys(firstHit)) + } + } + } + } + } + } + + return parseSearchResponse(body) +} + // SearchQuery represents a parsed search query with raw terms and field filters type SearchQuery struct { RawTerms []string @@ -154,7 +231,7 @@ func ParseSearchQuery(searchStr string) SearchQuery { // This is a field:value pair with dot notation fieldName := match[2] fieldValue := match[3] - + // Add to the array of values for this field query.FieldFilters[fieldName] = append(query.FieldFilters[fieldName], fieldValue) } else if match[4] != "" { @@ -164,7 +241,7 @@ func ParseSearchQuery(searchStr string) SearchQuery { // Simple field:value without dot notation fieldName := parts[0] fieldValue := parts[1] - + // Add to the array of values for this field query.FieldFilters[fieldName] = append(query.FieldFilters[fieldName], fieldValue) } else { @@ -198,7 +275,7 @@ func BuildTypesenseQuery(query SearchQuery) (string, map[string]string, error) { for _, value := range values { // Create the filter expression filterExpr := fmt.Sprintf("%s:%s", field, value) - + // Add to the corresponding field group fieldGroups[baseName] = append(fieldGroups[baseName], filterExpr) } @@ -234,44 +311,44 @@ func parseSearchResponse(responseBody []byte) ([]nostr.Event, error) { // Debug: Print the raw response structure fmt.Printf("Search response found %d hits\n", searchResponse.Found) - + nostrResults := make([]nostr.Event, 0, len(searchResponse.Hits)) for i, hit := range searchResponse.Hits { // Debug: Print hit structure information fmt.Printf("Processing hit %d, keys: %v\n", i, getMapKeys(hit)) - + // Check if document exists in the hit docRaw, exists := hit["document"] if !exists { fmt.Printf("Warning: hit %d has no 'document' field\n", i) continue // Skip this hit } - + // Extract document directly as a map[string]interface{} docMap, ok := docRaw.(map[string]interface{}) if !ok { fmt.Printf("Warning: hit %d document is not a map, type: %T\n", i, docRaw) continue // Skip this hit } - + // Debug: Print document keys fmt.Printf("Document keys: %v\n", getMapKeys(docMap)) - + // Check for EventRaw field directly eventRawVal, hasEventRaw := docMap["eventRaw"] if !hasEventRaw { fmt.Printf("Warning: document has no 'eventRaw' field\n") continue // Skip this document } - + // Try to extract EventRaw as string eventRawStr, ok := eventRawVal.(string) if !ok { fmt.Printf("Warning: eventRaw is not a string, type: %T\n", eventRawVal) continue // Skip this document } - + // Convert the EventRaw string to a Nostr event nostrEvent, err := StringifiedJSONToNostrEvent(eventRawStr) if err != nil { diff --git a/typesense30142/query_test.go b/typesense30142/query_test.go index d9c2abd..1e5ae05 100644 --- a/typesense30142/query_test.go +++ b/typesense30142/query_test.go @@ -1,2 +1,106 @@ package typesense30142 +import ( + "testing" + + "github.com/nbd-wtf/go-nostr" +) + +func TestQueryEventsWithLimit(t *testing.T) { + // Test that limit is properly handled + tests := []struct { + name string + filter nostr.Filter + expectedLimit int + }{ + { + name: "No limit specified - should default to 100", + filter: nostr.Filter{ + Search: "", + }, + expectedLimit: 100, + }, + { + name: "Limit of 2 specified", + filter: nostr.Filter{ + Search: "", + Limit: 2, + }, + expectedLimit: 2, + }, + { + name: "Limit of 50 specified", + filter: nostr.Filter{ + Search: "test", + Limit: 50, + }, + expectedLimit: 50, + }, + { + name: "Zero limit - should default to 100", + filter: nostr.Filter{ + Search: "test", + Limit: 0, + }, + expectedLimit: 100, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test the limit logic + limit := 100 + if tt.filter.Limit > 0 { + limit = tt.filter.Limit + } + + if limit != tt.expectedLimit { + t.Errorf("Expected limit %d, got %d", tt.expectedLimit, limit) + } + }) + } +} + +func TestBuildTypesenseQueryWithEmptySearch(t *testing.T) { + // Test that empty search strings are handled correctly + query := ParseSearchQuery("") + mainQuery, params, err := BuildTypesenseQuery(query) + + if err != nil { + t.Errorf("BuildTypesenseQuery failed: %v", err) + } + + // Empty search should result in empty main query + if mainQuery != "" { + t.Errorf("Expected empty main query, got: %s", mainQuery) + } + + // Should have no filter parameters for empty search + if len(params) != 0 { + t.Errorf("Expected no filter params for empty search, got: %v", params) + } +} + +func TestSearchResourcesWithLimitHandlesEmptySearch(t *testing.T) { + // Test that SearchResourcesWithLimit properly handles empty search by using wildcard + query := ParseSearchQuery("") + mainQuery, _, err := BuildTypesenseQuery(query) + + if err != nil { + t.Errorf("BuildTypesenseQuery failed: %v", err) + } + + // Empty search should result in empty main query initially + if mainQuery != "" { + t.Errorf("Expected empty main query from ParseSearchQuery, got: %s", mainQuery) + } + + // SearchResourcesWithLimit should convert empty query to "*" + if mainQuery == "" { + mainQuery = "*" + } + + if mainQuery != "*" { + t.Errorf("Expected wildcard query '*', got: %s", mainQuery) + } +}