mostly functionnal, poorly tested
This commit is contained in:
ycc
206
models/storage.go
Normal file
206
models/storage.go
Normal file
@@ -0,0 +1,206 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/beego/beego/logs"
|
||||
"github.com/go-redis/redis"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
type Storage struct {
|
||||
ZincUrl string
|
||||
ZincLogin string
|
||||
ZincPassword string
|
||||
RedisUrl string
|
||||
RedisPassword string
|
||||
}
|
||||
|
||||
func LoadPeersJson(filename string) []Peer {
|
||||
var peers []Peer
|
||||
content, err := os.ReadFile("./peers.json")
|
||||
if err != nil {
|
||||
logs.Error("Error when opening file: ", err)
|
||||
}
|
||||
err = json.Unmarshal(content, &peers)
|
||||
if err != nil {
|
||||
logs.Error("Error during Unmarshal(): ", err)
|
||||
}
|
||||
return peers
|
||||
}
|
||||
|
||||
func (s *Storage) ImportData(peers []Peer) error {
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: s.RedisUrl,
|
||||
Password: s.RedisPassword, // no password set
|
||||
DB: 0, // use default DB
|
||||
})
|
||||
var indexedPeers []map[string]interface{}
|
||||
for _, p := range peers {
|
||||
// Creating data block for indexing
|
||||
indexedPeer := make(map[string]interface{})
|
||||
indexedPeer["_id"] = p.PeerId
|
||||
indexedPeer["name"] = p.Name
|
||||
indexedPeer["keywords"] = p.Keywords
|
||||
indexedPeer["name"] = p.Name
|
||||
indexedPeer["entityname"] = p.EntityName
|
||||
indexedPeer["entitytype"] = p.EntityType
|
||||
indexedPeer["activity"] = p.Activity
|
||||
indexedPeer["address"] = p.Address
|
||||
indexedPeer["postcode"] = p.Postcode
|
||||
indexedPeer["city"] = p.City
|
||||
indexedPeer["country"] = p.Country
|
||||
indexedPeer["description"] = p.Description
|
||||
indexedPeer["apiurl"] = p.ApiUrl
|
||||
indexedPeer["website"] = p.Website
|
||||
indexedPeers = append(indexedPeers, indexedPeer)
|
||||
// Adding peer to Redis (fast retieval and status updates)
|
||||
jsonp, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = rdb.Set("peer:"+p.PeerId, jsonp, 0).Err()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
bulk := map[string]interface{}{"index": "peers", "records": indexedPeers}
|
||||
raw, err := json.Marshal(bulk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequest("POST", s.ZincUrl+"/api/_bulkv2", strings.NewReader(string(raw)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.SetBasicAuth(s.ZincLogin, s.ZincPassword)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
log.Println(resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Storage) FindPeers(queryString string) ([]Peer, error) {
|
||||
var peers []Peer
|
||||
query := `{
|
||||
"search_type": "match",
|
||||
"query":
|
||||
{
|
||||
"term": "` + queryString + `",
|
||||
"start_time": "2020-06-02T14:28:31.894Z",
|
||||
"end_time": "2029-12-02T15:28:31.894Z"
|
||||
},
|
||||
"from": 0,
|
||||
"max_results": 100,
|
||||
"_source": []
|
||||
}`
|
||||
req, err := http.NewRequest("POST", s.ZincUrl+"/api/peers/_search", strings.NewReader(query))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
req.SetBasicAuth(s.ZincLogin, s.ZincPassword)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
log.Println(resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value := gjson.Get(string(body), "hits.hits")
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: s.RedisUrl,
|
||||
Password: s.RedisPassword, // no password set
|
||||
DB: 0, // use default DB
|
||||
})
|
||||
for _, v := range value.Array() {
|
||||
peerBytes, err := rdb.Get("peer:" + v.Get("_id").Str).Bytes()
|
||||
if err != nil {
|
||||
logs.Error(err)
|
||||
} else {
|
||||
var p Peer
|
||||
err = json.Unmarshal(peerBytes, &p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
peers = append(peers, p)
|
||||
}
|
||||
}
|
||||
return peers, nil
|
||||
}
|
||||
|
||||
func (s *Storage) GetPeer(uid string) (*Peer, error) {
|
||||
var peer Peer
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: s.RedisUrl,
|
||||
Password: s.RedisPassword, // no password set
|
||||
DB: 0, // use default DB
|
||||
})
|
||||
peerBytes, err := rdb.Get("peer:" + uid).Bytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
err = json.Unmarshal(peerBytes, &peer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &peer, nil
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (s *Storage) DeletePeer(uid string) error {
|
||||
// Removing from Redis
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Addr: s.RedisUrl,
|
||||
Password: s.RedisPassword, // no password set
|
||||
DB: 0, // use default DB
|
||||
})
|
||||
err := rdb.Unlink("peer:" + uid).Err()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Removing from Index
|
||||
req, err := http.NewRequest("DELETE", s.ZincUrl+"/api/peers/_doc"+uid, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.SetBasicAuth(s.ZincLogin, s.ZincPassword)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
log.Println(resp.StatusCode)
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user