oc-discovery/models/storage.go

package models

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/beego/beego/logs"
	"github.com/go-redis/redis"
	"github.com/tidwall/gjson"
)

type Storage struct {
	ZincUrl       string
	ZincLogin     string
	ZincPassword  string
	RedisUrl      string
	RedisPassword string
}

func LoadPeersJson(filename string) []Peer {
	var peers []Peer
	content, err := os.ReadFile("./peers.json")
	if err != nil {
		logs.Error("Error when opening file: ", err)
	}
	err = json.Unmarshal(content, &peers)
	if err != nil {
		logs.Error("Error during Unmarshal(): ", err)
	}
	return peers
}

func (s *Storage) ImportData(peers []Peer) error {
	rdb := redis.NewClient(&redis.Options{
		Addr:     s.RedisUrl,
		Password: s.RedisPassword, // no password set
		DB:       0,               // use default DB
	})
	var indexedPeers []map[string]interface{}
	for _, p := range peers {
		// Creating data block for indexing
		indexedPeer := make(map[string]interface{})
		indexedPeer["_id"] = p.PeerId
		indexedPeer["name"] = p.Name
		indexedPeer["keywords"] = p.Keywords
		indexedPeer["name"] = p.Name
		indexedPeer["entityname"] = p.EntityName
		indexedPeer["entitytype"] = p.EntityType
		indexedPeer["activity"] = p.Activity
		indexedPeer["address"] = p.Address
		indexedPeer["postcode"] = p.Postcode
		indexedPeer["city"] = p.City
		indexedPeer["country"] = p.Country
		indexedPeer["description"] = p.Description
		indexedPeer["apiurl"] = p.ApiUrl
		indexedPeer["website"] = p.Website
		indexedPeers = append(indexedPeers, indexedPeer)
		// Adding peer to Redis (fast retieval and status updates)
		jsonp, err := json.Marshal(p)
		if err != nil {
			return err
		}
		err = rdb.Set("peer:"+p.PeerId, jsonp, 0).Err()
		if err != nil {
			return err
		}
	}
	bulk := map[string]interface{}{"index": "peers", "records": indexedPeers}
	raw, err := json.Marshal(bulk)
	if err != nil {
		return err
	}
	req, err := http.NewRequest("POST", s.ZincUrl+"/api/_bulkv2", strings.NewReader(string(raw)))
	if err != nil {
		return err
	}
	req.SetBasicAuth(s.ZincLogin, s.ZincPassword)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	log.Println(resp.StatusCode)
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	fmt.Println(string(body))
	return nil
}

func (s *Storage) FindPeers(queryString string) ([]Peer, error) {
	var peers []Peer
	query := `{
        "search_type": "match",
        "query":
        {
            "term": "` + queryString + `",
            "start_time": "2020-06-02T14:28:31.894Z",
            "end_time": "2029-12-02T15:28:31.894Z"
        },
        "from": 0,
        "max_results": 100,
        "_source": []
    }`
	req, err := http.NewRequest("POST", s.ZincUrl+"/api/peers/_search", strings.NewReader(query))
	if err != nil {
		log.Fatal(err)
	}
	req.SetBasicAuth(s.ZincLogin, s.ZincPassword)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	log.Println(resp.StatusCode)
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	value := gjson.Get(string(body), "hits.hits")
	rdb := redis.NewClient(&redis.Options{
		Addr:     s.RedisUrl,
		Password: s.RedisPassword, // no password set
		DB:       0,               // use default DB
	})
	for _, v := range value.Array() {
		peerBytes, err := rdb.Get("peer:" + v.Get("_id").Str).Bytes()
		if err != nil {
			logs.Error(err)
		} else {
			var p Peer
			err = json.Unmarshal(peerBytes, &p)
			if err != nil {
				return nil, err
			}
			peers = append(peers, p)
		}
	}
	return peers, nil
}

func (s *Storage) GetPeer(uid string) (*Peer, error) {
	var peer Peer
	rdb := redis.NewClient(&redis.Options{
		Addr:     s.RedisUrl,
		Password: s.RedisPassword, // no password set
		DB:       0,               // use default DB
	})
	peerBytes, err := rdb.Get("peer:" + uid).Bytes()
	if err != nil {
		return nil, err
	} else {
		err = json.Unmarshal(peerBytes, &peer)
		if err != nil {
			return nil, err
		}
		return &peer, nil
	}

}

func (s *Storage) DeletePeer(uid string) error {
	// Removing from Redis
	rdb := redis.NewClient(&redis.Options{
		Addr:     s.RedisUrl,
		Password: s.RedisPassword, // no password set
		DB:       0,               // use default DB
	})
	err := rdb.Unlink("peer:" + uid).Err()
	if err != nil {
		return err
	}
	// Removing from Index
	req, err := http.NewRequest("DELETE", s.ZincUrl+"/api/peers/_doc"+uid, nil)
	if err != nil {
		return err
	}
	req.SetBasicAuth(s.ZincLogin, s.ZincPassword)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	log.Println(resp.StatusCode)
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	fmt.Println(string(body))
	return nil
}