edufeed-org/typesense-indexer
2
0
Fork 0
mirror of https://github.com/edufeed-org/typesense-indexer.git synced 2025-12-07 23:34:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

use nostr-clj; add learningResourceType to be indexed

Browse source
This commit is contained in:
@s.roertgen 2024-12-02 16:20:25 +01:00
parent 8409c811d7
commit 022313ccc3
5 changed files with 42 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.clj-kondo/imports/http-kit/http-kit/config.edn Normal file
View file

@ -0,0 +1,3 @@
{:hooks
{:analyze-call {org.httpkit.server/with-channel httpkit.with-channel/with-channel}}}

16
.clj-kondo/imports/http-kit/http-kit/httpkit/with_channel.clj Normal file
View file

@ -0,0 +1,16 @@
(ns httpkit.with-channel
(:require [clj-kondo.hooks-api :as api]))
(defn with-channel [{node :node}]
(let [[request channel & body] (rest (:children node))]
(when-not (and request channel) (throw (ex-info "No request or channel provided" {})))
(when-not (api/token-node? channel) (throw (ex-info "Missing channel argument" {})))
(let [new-node
(api/list-node
(list*
(api/token-node 'let)
(api/vector-node [channel (api/vector-node [])])
request
body))]
{:node new-node})))

2
deps.edn
View file

@ -1,5 +1,5 @@
{:deps {com.stuartsierra/component {:mvn/version "1.1.0"}
hato/hato {:mvn/version "1.0.0"}
http-kit/http-kit {:mvn/version "2.8.0"}
nostr-clj/nostr-clj {:local/root "/home/laoc/coding/test/nostr-clj"}
net.clojars.laoc/nostr {:local/root "/home/laoc/coding/nostr-clj"}
cheshire/cheshire {:mvn/version "5.10.0"}}}

10
src/typesense_indexer/components/typesense.clj
View file

@ -2,7 +2,7 @@
(:require [com.stuartsierra.component :as component]
[cheshire.core :as json]
[org.httpkit.client :as http]
[nostr-clj.edufeed :as nostr]))
[nostr.edufeed :as nostr]))
;; Typesense Client Configuration
(def typesense-url "http://localhost:8108")
@ -16,7 +16,9 @@
{:name "description" :type "string" :optional true}
{:name "keywords" :type "string[]" :optional true}
{:name "about.id" :type "string[]" :optional true}
{:name "about.prefLabel" :type "object[]" :optional true}]})
{:name "about.prefLabel" :type "object[]" :optional true}
{:name "learningResourceType.id" :type "string[]" :optional true}
{:name "learningResourceType.prefLabel" :type "object[]" :optional true}]})
(defn insert-collection []
(http/post (str typesense-url "/collections")
@ -38,9 +40,7 @@
;; Utility function to insert a document into Typesense
(defn insert-to-typesense [collection event]
(let [url (str typesense-url "/collections/" collection "/documents?dirty_values=drop&action=upsert")
doc (nostr/convert-30142-to-nostr-amb event)
; _ (println "converted doc " doc)
]
doc (nostr/convert-30142-to-nostr-amb event false)]
(http/post url
{:body (json/encode doc)
:headers {"X-TYPESENSE-API-KEY" typesense-api-key

42
src/typesense_indexer/components/websocket.clj
View file

@ -2,27 +2,24 @@
(:require [hato.websocket :as ws]
[com.stuartsierra.component :as component]
[cheshire.core :as json]
[typesense-indexer.components.typesense :as typesense])
[typesense-indexer.components.typesense :as typesense]
[nostr.core :as nostr])
(:import [java.nio CharBuffer]))
(defn init-request-30142 [wc last-event newest-event reason]
(case reason
"eose" (do
(ws/send! wc (json/generate-string ["CLOSE" "RAND"]))
(ws/send! wc (json/generate-string ["REQ" "RANDNEW" {:kinds [30142]
:since (:created_at @newest-event)}])))
"reconnect" (ws/send! wc (json/generate-string ["REQ" "RAND" {:kinds [1]
:limit 4
:until (:created_at @last-event)}]))
"init" (ws/send! wc (json/generate-string ["REQ" "RAND" {:kinds [30142]
:limit 6000}]))))
(nostr/send! wc ["CLOSE" "RAND"])
(nostr/send! wc ["REQ" "RANDNEW" {:kinds [30142]
:since (:created_at @newest-event)}]))
"reconnect" (nostr/send! wc ["REQ" "RAND" {:kinds [1]
:limit 4
:until (:created_at @last-event)}])
"init" (nostr/send! wc ["REQ" "RAND" {:kinds [30142]
:limit 6000}])))
(defn on-message-handler [last-parsed-event newest-event ws msg last?]
(let [msg-str (if (instance? CharBuffer msg)
(str msg)
msg)
parsed (json/parse-string msg-str true)
event (nth parsed 2 nil)]
(defn on-message-handler [ws parsed last-parsed-event newest-event]
(let [event (nth parsed 2 nil)]
(println (first parsed) (:id event))
(when (and (= "EOSE" (first parsed))
(not= (:created_at @last-parsed-event) (:created_at @newest-event)))
@ -35,15 +32,10 @@
(reset! last-parsed-event event)))))
(defn create-websocket [url on-close-handler last-parsed-event newest-event]
@(ws/websocket url
{:on-open (fn [ws]
(println "Opened connection to url " url))
:on-message (fn [ws msg last?]
(on-message-handler last-parsed-event newest-event ws msg last?))
:on-close (fn [ws status reason]
;; status 1000 für ws/close!, status 1006 bei connection lost
(println "closed connection. status: " status " reason " reason)
(on-close-handler status))}))
(nostr/connect url
{:on-open-handler (fn [ws] (println "Opened connection to url " url))
:on-message-handler (fn [ws msg] (on-message-handler ws msg last-parsed-event newest-event))
:on-close-handler (fn [ws status reason] (on-close-handler status))}))
(defrecord WebsocketConnection [url connection]
component/Lifecycle
@ -75,7 +67,7 @@
(stop [component]
(println ";; Stopping WebsocketConnection for url " url)
(when-let [wc (:connection component)]
(ws/close! wc))
(nostr/close! wc))
(assoc component :connection nil)))
(defn new-websocket-connection [url]