mirror of
https://github.com/edufeed-org/amb-relay.git
synced 2025-12-07 15:24:33 +00:00
Lot of refactor, use .env and dedicated eventstore
This commit is contained in:
@s.roertgen
parent
68d3012dba
commit
d3b6a3f04a
8 changed files with 47 additions and 822 deletions
8
.env.example
Normal file
8
.env.example
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
NAME=""
|
||||
PUBKEY=""
|
||||
DESCRIPTION=""
|
||||
ICON=""
|
||||
TS_APIKEY=""
|
||||
TS_HOST=""
|
||||
TS_COLLECTION=""
|
||||
|
||||
3
.gitignore
vendored
Normal file
3
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
typesense-data/
|
||||
.env
|
||||
|
||||
7
README.md
Normal file
7
README.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Typesense Relay for AMB Metadata
|
||||
|
||||
- copy `.env.example` to `.env`
|
||||
- add metadata to your relay and typesense connection info
|
||||
- start typesense with `docker compose up`
|
||||
- start relay with `go run .`
|
||||
|
||||
7
go.mod
7
go.mod
|
|
@ -5,9 +5,13 @@ go 1.24.1
|
|||
require (
|
||||
github.com/fiatjaf/eventstore v0.16.4
|
||||
github.com/fiatjaf/khatru v0.17.5
|
||||
github.com/nbd-wtf/go-nostr v0.51.7
|
||||
github.com/nbd-wtf/go-nostr v0.51.8
|
||||
)
|
||||
|
||||
require github.com/edufeed-org/eventstore v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace github.com/edufeed-org/eventstore => ../eventstore
|
||||
|
||||
require (
|
||||
fiatjaf.com/lib v0.2.0 // indirect
|
||||
github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect
|
||||
|
|
@ -28,6 +32,7 @@ require (
|
|||
github.com/fasthttp/websocket v1.5.12 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/google/flatbuffers v24.12.23+incompatible // indirect
|
||||
github.com/joho/godotenv v1.5.1 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -79,6 +79,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
|||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
|
|
@ -96,8 +98,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/nbd-wtf/go-nostr v0.51.7 h1:dGjtaaFQ1kA3H+vF8wt9a9WYl54K8C0JmVDf4cp+a4A=
|
||||
github.com/nbd-wtf/go-nostr v0.51.7/go.mod h1:d6+DfvMWYG5pA3dmNMBJd6WCHVDDhkXbHqvfljf0Gzg=
|
||||
github.com/nbd-wtf/go-nostr v0.51.8 h1:CIoS+YqChcm4e1L1rfMZ3/mIwTz4CwApM2qx7MHNzmE=
|
||||
github.com/nbd-wtf/go-nostr v0.51.8/go.mod h1:d6+DfvMWYG5pA3dmNMBJd6WCHVDDhkXbHqvfljf0Gzg=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
|
|
|
|||
74
main.go
74
main.go
|
|
@ -3,33 +3,33 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/fiatjaf/eventstore/badger"
|
||||
"github.com/edufeed-org/eventstore/typesense30142"
|
||||
"github.com/fiatjaf/khatru"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
const (
|
||||
apiKey string = "xyz"
|
||||
typesenseHost string = "http://localhost:8108"
|
||||
collectionName string = "amb-test"
|
||||
)
|
||||
|
||||
func main() {
|
||||
err := CheckOrCreateCollection("amb-test")
|
||||
// Load .env file
|
||||
err := godotenv.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to check/create collection: %v", err)
|
||||
fmt.Printf("Error loading .env file: %v", err)
|
||||
}
|
||||
|
||||
relay := khatru.NewRelay()
|
||||
relay.Info.Name = "my typesense relay"
|
||||
// tsRelay.Info.PubKey = "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798"
|
||||
relay.Info.Description = "this is the typesense custom relay"
|
||||
// tsRelay.Info.Icon = "https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fliquipedia.net%2Fcommons%2Fimages%2F3%2F35%2FSCProbe.jpg&f=1&nofb=1&ipt=0cbbfef25bce41da63d910e86c3c343e6c3b9d63194ca9755351bb7c2efa3359&ipo=images"
|
||||
dbts := badger.BadgerBackend{Path: "/tmp/khatru-badgern-tmp-2"}
|
||||
if err := dbts.Init(); err != nil {
|
||||
relay.Info.Name = os.Getenv("NAME")
|
||||
relay.Info.PubKey = os.Getenv("PUBKEY")
|
||||
relay.Info.Description = os.Getenv("DESCRIPTION")
|
||||
relay.Info.Icon = os.Getenv("ICON")
|
||||
db := typesense30142.TSBackend{
|
||||
ApiKey: os.Getenv("TS_APIKEY"),
|
||||
Host: os.Getenv("TS_HOST"),
|
||||
CollectionName: os.Getenv("TS_COLLECTION"),
|
||||
}
|
||||
if err := db.Init(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
|
|
@ -37,10 +37,9 @@ func main() {
|
|||
khatru.RequestAuth(ctx)
|
||||
})
|
||||
|
||||
relay.QueryEvents = append(relay.QueryEvents, handleQuery)
|
||||
relay.CountEvents = append(relay.CountEvents, handleCount)
|
||||
relay.DeleteEvent = append(relay.DeleteEvent, handleDelete)
|
||||
relay.ReplaceEvent = append(relay.ReplaceEvent, handleReplaceEvent)
|
||||
relay.QueryEvents = append(relay.QueryEvents, db.QueryEvents)
|
||||
relay.DeleteEvent = append(relay.DeleteEvent, db.DeleteEvent)
|
||||
relay.ReplaceEvent = append(relay.ReplaceEvent, db.ReplaceEvent)
|
||||
relay.Negentropy = true
|
||||
|
||||
relay.RejectEvent = append(relay.RejectEvent,
|
||||
|
|
@ -55,38 +54,3 @@ func main() {
|
|||
fmt.Println("running on :3334")
|
||||
http.ListenAndServe(":3334", relay)
|
||||
}
|
||||
|
||||
func handleQuery(ctx context.Context, filter nostr.Filter) (chan *nostr.Event, error) {
|
||||
ch := make(chan *nostr.Event)
|
||||
|
||||
nostrs, err := SearchResources(collectionName, filter.Search)
|
||||
if err != nil {
|
||||
log.Printf("Search failed: %v", err)
|
||||
return ch, err
|
||||
}
|
||||
|
||||
go func() {
|
||||
for _, evt := range nostrs {
|
||||
ch <- &evt
|
||||
}
|
||||
close(ch)
|
||||
}()
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func handleCount(ctx context.Context, filter nostr.Filter) (int64, error) {
|
||||
CountEvents(collectionName, filter)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func handleDelete(ctx context.Context, event *nostr.Event) error {
|
||||
fmt.Println("delete event", event)
|
||||
DeleteNostrEvent(collectionName, event)
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleReplaceEvent(ctx context.Context, event *nostr.Event) error {
|
||||
IndexNostrEvent(collectionName, event)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
327
nostramb.go
327
nostramb.go
|
|
@ -1,327 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
// BaseEntity contains common fields used across many entity types
|
||||
type BaseEntity struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
type ControlledVocabulary struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
ID string `json:"id"`
|
||||
PrefLabel string `json:"prefLabel"`
|
||||
InLanguage string `json:"inLanguage,omitempty"`
|
||||
}
|
||||
|
||||
// LanguageEntity adds language support to entities
|
||||
type LanguageEntity struct {
|
||||
InLanguage string `json:"inLanguage,omitempty"`
|
||||
}
|
||||
|
||||
// LabeledEntity adds prefLabel to entities
|
||||
type LabeledEntity struct {
|
||||
PrefLabel string `json:"prefLabel,omitempty"`
|
||||
}
|
||||
|
||||
// About represents a topic or subject
|
||||
type About struct {
|
||||
ControlledVocabulary
|
||||
}
|
||||
|
||||
// Creator represents the creator of content
|
||||
type Creator struct {
|
||||
BaseEntity
|
||||
Affiliation *Affiliation `json:"affiliation,omitempty"`
|
||||
HonoricPrefix string `json:"honoricPrefix,omitempty"`
|
||||
}
|
||||
|
||||
// Contributor represents someone who contributed to the content
|
||||
type Contributor struct {
|
||||
BaseEntity
|
||||
HonoricPrefix string `json:"honoricPrefix,omitempty"`
|
||||
}
|
||||
|
||||
// Publisher represents the publisher of the content
|
||||
type Publisher struct {
|
||||
BaseEntity
|
||||
}
|
||||
|
||||
// Funder represents an entity that funded the content
|
||||
type Funder struct {
|
||||
BaseEntity
|
||||
}
|
||||
|
||||
// Affiliation represents an organization affiliation
|
||||
type Affiliation struct {
|
||||
BaseEntity
|
||||
}
|
||||
|
||||
// ConditionsOfAccess represents access conditions
|
||||
type ConditionsOfAccess struct {
|
||||
ControlledVocabulary
|
||||
}
|
||||
|
||||
// LearningResourceType categorizes the learning resource
|
||||
type LearningResourceType struct {
|
||||
ControlledVocabulary
|
||||
}
|
||||
|
||||
// Audience represents the target audience
|
||||
type Audience struct {
|
||||
ControlledVocabulary
|
||||
}
|
||||
|
||||
// Teaches represents what the content teaches
|
||||
type Teaches struct {
|
||||
ID string `json:"id"`
|
||||
LabeledEntity
|
||||
LanguageEntity
|
||||
}
|
||||
|
||||
// Assesses represents what the content assesses
|
||||
type Assesses struct {
|
||||
ID string `json:"id"`
|
||||
LabeledEntity
|
||||
LanguageEntity
|
||||
}
|
||||
|
||||
// CompetencyRequired represents required competencies
|
||||
type CompetencyRequired struct {
|
||||
ID string `json:"id"`
|
||||
LabeledEntity
|
||||
LanguageEntity
|
||||
}
|
||||
|
||||
// EducationalLevel represents the educational level
|
||||
type EducationalLevel struct {
|
||||
ControlledVocabulary
|
||||
}
|
||||
|
||||
// InteractivityType represents the type of interactivity
|
||||
type InteractivityType struct {
|
||||
ControlledVocabulary
|
||||
}
|
||||
|
||||
// IsBasedOn represents a reference to source material
|
||||
type IsBasedOn struct {
|
||||
Type string `json:"type,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Creator *Creator `json:"creator,omitempty"`
|
||||
License *License `json:"license,omitempty"`
|
||||
}
|
||||
|
||||
// IsPartOf represents a parent relationship
|
||||
type IsPartOf struct {
|
||||
BaseEntity
|
||||
}
|
||||
|
||||
type HasPart struct {
|
||||
BaseEntity
|
||||
}
|
||||
|
||||
// Trailer represents a media trailer
|
||||
type Trailer struct {
|
||||
Type string `json:"type"`
|
||||
ContentUrl string `json:"contentUrl"`
|
||||
EncodingFormat string `json:"encodingFormat"`
|
||||
ContentSize string `json:"contentSize,omitempty"`
|
||||
Sha256 string `json:"sha256,omitempty"`
|
||||
EmbedUrl string `json:"embedUrl,omitempty"`
|
||||
Bitrate string `json:"bitrate,omitempty"`
|
||||
}
|
||||
|
||||
// License represents the content license
|
||||
type License struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// NostrMetadata contains Nostr-specific metadata
|
||||
type NostrMetadata struct {
|
||||
EventID string `json:"eventID"`
|
||||
EventKind int `json:"eventKind"`
|
||||
EventPubKey string `json:"eventPubKey"`
|
||||
EventSig string `json:"eventSignature"`
|
||||
EventCreatedAt nostr.Timestamp `json:"eventCreatedAt"`
|
||||
EventContent string `json:"eventContent"`
|
||||
EventRaw string `json:"eventRaw"`
|
||||
}
|
||||
|
||||
// AMBMetadata represents the full metadata structure
|
||||
type AMBMetadata struct {
|
||||
// Event ID
|
||||
ID string `json:"id"`
|
||||
// Document ID
|
||||
D string `json:"d"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
About []*About `json:"about,omitempty"`
|
||||
Keywords []string `json:"keywords,omitempty"`
|
||||
InLanguage []string `json:"inLanguage,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
Trailer []*Trailer `json:"trailer,omitempty"`
|
||||
|
||||
// Provenience
|
||||
Creator []*Creator `json:"creator,omitempty"`
|
||||
Contributor []*Contributor `json:"contributor,omitempty"`
|
||||
DateCreated string `json:"dateCreated,omitempty"`
|
||||
DatePublished string `json:"datePublished,omitempty"`
|
||||
DateModified string `json:"dateModified,omitempty"`
|
||||
Publisher []*Publisher `json:"publisher,omitempty"`
|
||||
Funder []*Funder `json:"funder,omitempty"`
|
||||
|
||||
// Costs and Rights
|
||||
IsAccessibleForFree bool `json:"isAccessibleForFree,omitempty"`
|
||||
License *License `json:"license,omitempty"`
|
||||
ConditionsOfAccess *ConditionsOfAccess `json:"conditionsOfAccess,omitempty"`
|
||||
|
||||
// Educational metadata
|
||||
LearningResourceType []*LearningResourceType `json:"learningResourceType,omitempty"`
|
||||
Audience []*Audience `json:"audience,omitempty"`
|
||||
Teaches []*Teaches `json:"teaches,omitempty"`
|
||||
Assesses []*Assesses `json:"assesses,omitempty"`
|
||||
CompetencyRequired []*CompetencyRequired `json:"competencyRequired,omitempty"`
|
||||
EducationalLevel []*EducationalLevel `json:"educationalLevel,omitempty"`
|
||||
InteractivityType *InteractivityType `json:"interactivityType,omitempty"`
|
||||
|
||||
// Relation
|
||||
IsBasedOn []*IsBasedOn `json:"isBasedOn,omitempty"`
|
||||
IsPartOf []*IsPartOf `json:"isPartOf,omitempty"`
|
||||
HasPart []*HasPart `json:"hasPart,omitempty"`
|
||||
|
||||
// Technical
|
||||
Duration string `json:"duration,omitempty"`
|
||||
// TODO Encoding ``
|
||||
// TODO Caption
|
||||
|
||||
// Nostr integration
|
||||
NostrMetadata `json:",inline"`
|
||||
}
|
||||
|
||||
// converts a nostr event to stringified JSON
|
||||
func eventToStringifiedJSON(event *nostr.Event) (string, error) {
|
||||
jsonData, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
jsonString := string(jsonData)
|
||||
return jsonString, err
|
||||
}
|
||||
|
||||
// NostrToAMB converts a Nostr event of kind 30142 to AMB metadata
|
||||
func NostrToAMB(event *nostr.Event) (*AMBMetadata, error) {
|
||||
eventRaw, _ := eventToStringifiedJSON(event)
|
||||
|
||||
amb := &AMBMetadata{
|
||||
Type: "LearningResource",
|
||||
NostrMetadata: NostrMetadata{
|
||||
EventID: event.ID,
|
||||
EventPubKey: event.PubKey,
|
||||
EventContent: event.Content,
|
||||
EventCreatedAt: event.CreatedAt,
|
||||
EventKind: event.Kind,
|
||||
EventSig: event.Sig,
|
||||
EventRaw: eventRaw,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tag := range event.Tags {
|
||||
if len(tag) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
// TODO alle Attribute durchgehen für das parsen
|
||||
switch tag[0] {
|
||||
case "d":
|
||||
if len(tag) >= 2 {
|
||||
amb.ID = event.ID
|
||||
amb.D = tag[1]
|
||||
}
|
||||
case "name":
|
||||
if len(tag) >= 2 {
|
||||
amb.Name = tag[1]
|
||||
}
|
||||
case "description":
|
||||
if len(tag) >= 2 {
|
||||
amb.Description = tag[1]
|
||||
}
|
||||
case "creator":
|
||||
if len(tag) >= 2 {
|
||||
creator := &Creator{}
|
||||
creator.Name = tag[1]
|
||||
if len(tag) >= 3 {
|
||||
creator.ID = tag[2]
|
||||
}
|
||||
if len(tag) >= 4 {
|
||||
creator.Type = tag[3]
|
||||
}
|
||||
|
||||
amb.Creator = append(amb.Creator, creator)
|
||||
}
|
||||
case "image":
|
||||
if len(tag) >= 2 {
|
||||
amb.Image = tag[1]
|
||||
}
|
||||
case "about":
|
||||
if len(tag) >= 3 {
|
||||
subject := &About{}
|
||||
subject.PrefLabel = tag[1]
|
||||
subject.InLanguage = tag[2]
|
||||
if len(tag) >= 4 {
|
||||
subject.ID = tag[3]
|
||||
}
|
||||
amb.About = append(amb.About, subject)
|
||||
}
|
||||
case "learningResourceType":
|
||||
if len(tag) >= 3 {
|
||||
lrt := &LearningResourceType{}
|
||||
lrt.PrefLabel = tag[1]
|
||||
lrt.InLanguage = tag[2]
|
||||
if len(tag) >= 4 {
|
||||
lrt.ID = tag[3]
|
||||
}
|
||||
amb.LearningResourceType = append(amb.LearningResourceType, lrt)
|
||||
}
|
||||
case "inLanguage":
|
||||
if len(tag) >= 2 {
|
||||
amb.InLanguage = append(amb.InLanguage, tag[1])
|
||||
}
|
||||
case "keywords":
|
||||
if len(tag) >= 2 {
|
||||
amb.Keywords = tag[1:]
|
||||
}
|
||||
case "license":
|
||||
if len(tag) >= 3 {
|
||||
amb.License = &License{}
|
||||
amb.License.ID = tag[1]
|
||||
amb.License.Name = tag[2]
|
||||
|
||||
}
|
||||
case "datePublished":
|
||||
if len(tag) >= 2 {
|
||||
amb.DatePublished = tag[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return amb, nil
|
||||
}
|
||||
|
||||
// converts a stringified JSON event to a nostr.Event
|
||||
func StringifiedJSONToNostrEvent(jsonString string) (nostr.Event, error) {
|
||||
var event nostr.Event
|
||||
err := json.Unmarshal([]byte(jsonString), &event)
|
||||
if err != nil {
|
||||
return nostr.Event{}, err
|
||||
}
|
||||
return event, nil
|
||||
}
|
||||
437
typesense.go
437
typesense.go
|
|
@ -1,437 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
)
|
||||
|
||||
type CollectionSchema struct {
|
||||
Name string `json:"name"`
|
||||
Fields []Field `json:"fields"`
|
||||
DefaultSortingField string `json:"default_sorting_field"`
|
||||
EnableNestedFields bool `json:"enable_nested_fields"`
|
||||
}
|
||||
|
||||
type Field struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Facet bool `json:"facet,omitempty"`
|
||||
Optional bool `json:"optional,omitempty"`
|
||||
}
|
||||
|
||||
type SearchResponse struct {
|
||||
Found int `json:"found"`
|
||||
Hits []map[string]any `json:"hits"`
|
||||
Page int `json:"page"`
|
||||
Request map[string]any `json:"request"`
|
||||
}
|
||||
|
||||
// CheckOrCreateCollection checks if a collection exists and creates it if it doesn't
|
||||
func CheckOrCreateCollection(collectionName string) error {
|
||||
exists, err := collectionExists(collectionName)
|
||||
if err != nil {
|
||||
log.Fatalf("Error checking collection: %v", err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
fmt.Printf("Collection %s does not exist. Creating...\n", collectionName)
|
||||
if err := createCollection(collectionName); err != nil {
|
||||
log.Fatalf("Error creating collection: %v", err)
|
||||
}
|
||||
fmt.Printf("Collection %s created successfully\n", collectionName)
|
||||
} else {
|
||||
fmt.Printf("Collection %s already exists\n", collectionName)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func collectionExists(name string) (bool, error) {
|
||||
url := fmt.Sprintf("%s/collections/%s", typesenseHost, name)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
req.Header.Set("X-TYPESENSE-API-KEY", apiKey)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// 404 means collection doesn't exist
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Any status code other than 200 is an error
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return false, fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// create a typesense collection
|
||||
func createCollection(name string) error {
|
||||
schema := CollectionSchema{
|
||||
Name: name,
|
||||
Fields: []Field{
|
||||
// Base information
|
||||
{Name: "id", Type: "string"},
|
||||
{Name: "d", Type: "string"},
|
||||
{Name: "type", Type: "string"},
|
||||
{Name: "name", Type: "string"},
|
||||
{Name: "description", Type: "string", Optional: true},
|
||||
{Name: "about", Type: "object[]", Optional: true},
|
||||
{Name: "keywords", Type: "string[]", Optional: true},
|
||||
{Name: "inLanguage", Type: "string[]", Optional: true},
|
||||
{Name: "image", Type: "string", Optional: true},
|
||||
{Name: "trailer", Type: "object[]", Optional: true},
|
||||
|
||||
// Provenience
|
||||
{Name: "creator", Type: "object[]", Optional: true},
|
||||
{Name: "contributor", Type: "object[]", Optional: true},
|
||||
{Name: "dateCreated", Type: "string", Optional: true},
|
||||
{Name: "datePublished", Type: "string", Optional: true},
|
||||
{Name: "dateModified", Type: "string", Optional: true},
|
||||
{Name: "publisher", Type: "object[]", Optional: true},
|
||||
{Name: "funder", Type: "object[]", Optional: true},
|
||||
|
||||
// Costs and Rights
|
||||
{Name: "isAccessibleForFree", Type: "bool", Optional: true},
|
||||
{Name: "license", Type: "object", Optional: true},
|
||||
{Name: "conditionsOfAccess", Type: "object", Optional: true},
|
||||
|
||||
// Educational Metadata
|
||||
{Name: "learningResourceType", Type: "object[]", Optional: true},
|
||||
{Name: "audience", Type: "object[]", Optional: true},
|
||||
{Name: "teaches", Type: "object[]", Optional: true},
|
||||
{Name: "assesses", Type: "object[]", Optional: true},
|
||||
{Name: "competencyRequired", Type: "object[]", Optional: true},
|
||||
{Name: "educationalLevel", Type: "object[]", Optional: true},
|
||||
{Name: "interactivityType", Type: "object", Optional: true},
|
||||
|
||||
// Relation
|
||||
{Name: "isBasedOn", Type: "object[]", Optional: true},
|
||||
{Name: "isPartOf", Type: "object[]", Optional: true},
|
||||
{Name: "hasPart", Type: "object[]", Optional: true},
|
||||
|
||||
// Technical
|
||||
{Name: "duration", Type: "string", Optional: true},
|
||||
|
||||
// Nostr Event
|
||||
{Name: "eventID", Type: "string"},
|
||||
{Name: "eventKind", Type: "int32"},
|
||||
{Name: "eventPubKey", Type: "string"},
|
||||
{Name: "eventSignature", Type: "string"},
|
||||
{Name: "eventCreatedAt", Type: "int64"},
|
||||
{Name: "eventContent", Type: "string"},
|
||||
{Name: "eventRaw", Type: "string"},
|
||||
},
|
||||
DefaultSortingField: "eventCreatedAt",
|
||||
EnableNestedFields: true,
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/collections", typesenseHost)
|
||||
|
||||
jsonData, err := json.Marshal(schema)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := makehttpRequest(url, http.MethodPost, jsonData)
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("failed to create collection, status: %d, body: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Makes an http request to typesense
|
||||
func makehttpRequest(url string, method string, reqBody []byte) (*http.Response, error) {
|
||||
req, err := http.NewRequest(method, url, bytes.NewBuffer(reqBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("X-TYPESENSE-API-KEY", apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// TODO Count events
|
||||
func CountEvents(collectionName string, filter nostr.Filter) (int64, error) {
|
||||
fmt.Println("filter", filter)
|
||||
// search by author
|
||||
// search by d-tag
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Delete a nostr event from the index
|
||||
func DeleteNostrEvent(collectionName string, event *nostr.Event) error {
|
||||
fmt.Println("deleting event")
|
||||
d := event.Tags.GetD()
|
||||
|
||||
url := fmt.Sprintf(
|
||||
"%s/collections/%s/documents?filter_by=d:=%s&&eventPubKey:=%s",
|
||||
typesenseHost, collectionName, d, event.PubKey)
|
||||
|
||||
resp, err := makehttpRequest(url, http.MethodDelete, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Any status code other than 200 is an error
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// IndexNostrEvent converts a Nostr event to AMB metadata and indexes it in Typesense
|
||||
func IndexNostrEvent(collectionName string, event *nostr.Event) error {
|
||||
ambData, err := NostrToAMB(event)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error converting Nostr event to AMB metadata: %v", err)
|
||||
}
|
||||
|
||||
// check if event is already there, if so replace it, else index it
|
||||
alreadyIndexed, err := eventAlreadyIndexed(collectionName, ambData)
|
||||
return indexDocument(collectionName, ambData, alreadyIndexed)
|
||||
}
|
||||
|
||||
func eventAlreadyIndexed(collectionName string, doc *AMBMetadata) (*nostr.Event, error) {
|
||||
url := fmt.Sprintf(
|
||||
"%s/collections/%s/documents/search?filter_by=d:=%s&&eventPubKey:=%s&q=&query_by=d,eventPubKey",
|
||||
typesenseHost, collectionName, doc.D, doc.EventPubKey)
|
||||
|
||||
resp, err := makehttpRequest(url, http.MethodGet, nil)
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Search for event failed, status: %d, body: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
events, err := parseSearchResponse(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error while parsing search response: %v", err)
|
||||
}
|
||||
|
||||
// Check if we found any events
|
||||
if len(events) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &events[0], nil
|
||||
}
|
||||
|
||||
// Index a document in Typesense
|
||||
func indexDocument(collectionName string, doc *AMBMetadata, alreadyIndexedEvent *nostr.Event) error {
|
||||
if alreadyIndexedEvent != nil {
|
||||
fmt.Println("deleting old event for new one")
|
||||
DeleteNostrEvent(collectionName, alreadyIndexedEvent)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/collections/%s/documents", typesenseHost, collectionName)
|
||||
|
||||
jsonData, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := makehttpRequest(url, http.MethodPost, jsonData)
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
|
||||
return fmt.Errorf("failed to index document, status: %d, body: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SearchQuery represents a parsed search query with raw terms and field filters
|
||||
type SearchQuery struct {
|
||||
RawTerms []string
|
||||
FieldFilters map[string]string
|
||||
}
|
||||
|
||||
// ParseSearchQuery parses a search string with support for quoted terms and field:value pairs
|
||||
func ParseSearchQuery(searchStr string) SearchQuery {
|
||||
var query SearchQuery
|
||||
query.RawTerms = []string{}
|
||||
query.FieldFilters = make(map[string]string)
|
||||
|
||||
// Regular expression to match quoted strings and field:value pairs
|
||||
// This regex handles:
|
||||
// 1. Quoted strings (preserving spaces and everything inside)
|
||||
// 2. Field:value pairs
|
||||
// 3. Regular words
|
||||
re := regexp.MustCompile(`"([^"]+)"|(\S+\.\S+):(\S+)|(\S+)`)
|
||||
matches := re.FindAllStringSubmatch(searchStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if match[1] != "" {
|
||||
// This is a quoted string, add it to raw terms
|
||||
query.RawTerms = append(query.RawTerms, match[1])
|
||||
} else if match[2] != "" && match[3] != "" {
|
||||
// This is a field:value pair
|
||||
fieldName := match[2]
|
||||
fieldValue := match[3]
|
||||
query.FieldFilters[fieldName] = fieldValue
|
||||
} else if match[4] != "" {
|
||||
// This is a regular word, check if it's a simple field:value
|
||||
parts := strings.SplitN(match[4], ":", 2)
|
||||
if len(parts) == 2 && !strings.Contains(parts[0], ".") {
|
||||
// Simple field:value without dot notation
|
||||
query.FieldFilters[parts[0]] = parts[1]
|
||||
} else {
|
||||
// Regular search term
|
||||
query.RawTerms = append(query.RawTerms, match[4])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
// BuildTypesenseQuery builds a Typesense search query from a parsed SearchQuery
|
||||
func BuildTypesenseQuery(query SearchQuery) (string, map[string]string, error) {
|
||||
// Join raw terms for the main query
|
||||
mainQuery := strings.Join(query.RawTerms, " ")
|
||||
|
||||
// Parameters for filter_by and other Typesense parameters
|
||||
params := make(map[string]string)
|
||||
|
||||
// Build filter expressions for field filters
|
||||
var filterExpressions []string
|
||||
|
||||
for field, value := range query.FieldFilters {
|
||||
// Handle special fields with dot notation
|
||||
if strings.Contains(field, ".") {
|
||||
parts := strings.SplitN(field, ".", 2)
|
||||
fieldName := parts[0]
|
||||
subField := parts[1]
|
||||
|
||||
filterExpressions = append(filterExpressions, fmt.Sprintf("%s.%s:%s", fieldName, subField, value))
|
||||
} else {
|
||||
filterExpressions = append(filterExpressions, fmt.Sprintf("%s:%s", field, value))
|
||||
}
|
||||
}
|
||||
|
||||
// Combine all filter expressions
|
||||
if len(filterExpressions) > 0 {
|
||||
params["filter_by"] = strings.Join(filterExpressions, " && ")
|
||||
}
|
||||
|
||||
return mainQuery, params, nil
|
||||
}
|
||||
|
||||
// SearchResources searches for resources and returns both the AMB metadata and converted Nostr events
|
||||
func SearchResources(collectionName, searchStr string) ([]nostr.Event, error) {
|
||||
parsedQuery := ParseSearchQuery(searchStr)
|
||||
|
||||
mainQuery, params, err := BuildTypesenseQuery(parsedQuery)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error building Typesense query: %v", err)
|
||||
}
|
||||
|
||||
// URL encode the main query
|
||||
encodedQuery := url.QueryEscape(mainQuery)
|
||||
|
||||
// Default fields to search in
|
||||
queryBy := "name,description"
|
||||
|
||||
// Start building the search URL
|
||||
searchURL := fmt.Sprintf("%s/collections/%s/documents/search?q=%s&query_by=%s",
|
||||
typesenseHost, collectionName, encodedQuery, queryBy)
|
||||
|
||||
// Add additional parameters
|
||||
for key, value := range params {
|
||||
searchURL += fmt.Sprintf("&%s=%s", key, url.QueryEscape(value))
|
||||
}
|
||||
|
||||
// Debug information
|
||||
fmt.Printf("Search URL: %s\n", searchURL)
|
||||
|
||||
resp, err := makehttpRequest(searchURL, http.MethodGet, nil)
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading response body: %v", err)
|
||||
}
|
||||
|
||||
// Check for errors
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("search failed with status code %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return parseSearchResponse(body)
|
||||
}
|
||||
|
||||
func parseSearchResponse(responseBody []byte) ([]nostr.Event, error) {
|
||||
var searchResponse SearchResponse
|
||||
if err := json.Unmarshal(responseBody, &searchResponse); err != nil {
|
||||
return nil, fmt.Errorf("error parsing search response: %v", err)
|
||||
}
|
||||
|
||||
nostrResults := make([]nostr.Event, 0, len(searchResponse.Hits))
|
||||
|
||||
for _, hit := range searchResponse.Hits {
|
||||
// Extract the document from the hit
|
||||
docMap, ok := hit["document"].(map[string]AMBMetadata)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid document format in search results")
|
||||
}
|
||||
|
||||
// Convert the map to AMB metadata
|
||||
docJSON, err := json.Marshal(docMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error marshaling document: %v", err)
|
||||
}
|
||||
|
||||
var ambData AMBMetadata
|
||||
if err := json.Unmarshal(docJSON, &ambData); err != nil {
|
||||
return nil, fmt.Errorf("error unmarshaling to AMBMetadata: %v", err)
|
||||
}
|
||||
|
||||
// Convert the AMB metadata to a Nostr event
|
||||
nostrEvent, err := StringifiedJSONToNostrEvent(ambData.EventRaw)
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to convert AMB to Nostr: %v\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
nostrResults = append(nostrResults, nostrEvent)
|
||||
}
|
||||
|
||||
// Print the number of results for logging
|
||||
fmt.Printf("Found %d results\n",
|
||||
len(nostrResults))
|
||||
|
||||
return nostrResults, nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue