怎么实现一个分布式kv系统(1)-本地db

[TOC]

摘要

本节要实现的有2点

环境配置：远程服务器 + VS Code设置
实现类似redis的set和get功能
- 基于bolt.DB实现db模块
- 实现简单的http web模块

环境配置

1. 远程服务器[可选择]

远程服务器比较稳定，当然也是多折腾体验。

腾讯云服务器购买详情&一些奇奇怪怪的软件设置，请参考: 详细的配置信息

2. 配置VSCode

安装 Remote SSH插件

连接服务器

切换第一个tab，打开资源管理器，添加目录

3. 设置go mod

查看go版本和配置

[root@VM-24-14-centos ~]# go version
go version go1.17 linux/amd64
[root@VM-24-14-centos ~]# echo $GOPATH
/data/go
[root@VM-24-14-centos ~]# echo $GOROOT
/usr/local/go

vscode安装tools

设置vscode go的选项

"go.installDependenciesWhenBuilding": true,
"go.useCodeSnippetsOnFunctionSuggestWithoutType": true,
"go.autocompleteUnimportedPackages": true,
"go.gotoSymbol.includeImports": true,
"go.useCodeSnippetsOnFunctionSuggest": true,
"go.inferGopath": true,

编程实现

1. 导入boltdb

[root@VM-24-14-centos go]# mkdir kv-demo
[root@VM-24-14-centos go]# cd kv-demo
[root@VM-24-14-centos kv-demo]# go mod init example.com/kv-demo
[root@VM-24-14-centos kv-demo]# go mod tidy
[root@VM-24-14-centos kv-demo]# go mod vendor
[root@VM-24-14-centos kv-demo]# go run main.go

2. 设置flag

目前就一个参数db-location,后续可以添加

// main.go
var (
  dbLocation = flag.String("db-location", "my.db", "The path to the database location")
)

func main() {
  flag.Parse()
  db, err := bolt.Open(*dbLocation, 0600, nil)
  if err != nil {
    log.Fatal(err)
  }
  defer db.Close()
}

3. 创建HTTP API

先简单的mock http api

// main.go
http.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Get Called")
  })

http.HandleFunc("/set", func(w http.ResponseWriter, r *http.Request) {
  fmt.Fprintf(w, "Set called")
})

log.Fatal(http.ListenAndServe(":8090", nil))

[root@VM-24-14-centos kv-demo]# curl http://127.0.0.1:8090/set
Set called
[root@VM-24-14-centos kv-demo]# curl http://127.0.0.1:8090/get
Get Called

4. DB模块

目前位置db的功能都在main里面，我们分离出单独的db模块

创建DB
设置key
获取key
创建默认Bucket

func NewDatabase(dbLocation string) (db *Database, closeFunc func() error, err error) {
  boltDb, err := bolt.Open(dbLocation, 0600, nil)
  if err != nil {
    return nil, nil, err
  }
  db = &Database{db: boltDb}
  closeFunc = boltDb.Close

  if err := db.createDefaultBucket(); err != nil {
    return nil, nil, fmt.Errorf("create default bucket failed: %v", err)
  }

  return db, closeFunc, nil
}

func (db *Database) SetKey(key string, value []byte) error {
  return db.db.Update(func(tx *bolt.Tx) error {
    b := tx.Bucket(defaultBucket)
    return b.Put([]byte(key), value)
  })
}

func (db *Database) GetKey(key string) ([]byte, error) {
  var rc []byte
  err := db.db.View(func(tx *bolt.Tx) error {
    b := tx.Bucket(defaultBucket)
    rc = b.Get([]byte(key))
    return nil
  })
  return rc, err
}

func (db *Database) createDefaultBucket() error {
  return db.db.Update(func(tx *bolt.Tx) error {
    _, err := tx.CreateBucketIfNotExists([]byte(defaultBucket))
    return err
  })
}

5. 更新API

更新set & get http api来

http.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) {
  r.ParseForm()
  key := r.Form.Get("key")
  value, err := db.GetKey(key)
  fmt.Fprintf(w, "%q:%q; %v Get Called\n", key, value, err)
})


http.HandleFunc("/set", func(w http.ResponseWriter, r *http.Request) {
  r.ParseForm()
  key := r.Form.Get("key")
  value := r.Form.Get("value")
  err := db.SetKey(key, []byte(value))
  fmt.Fprintf(w, "err: %v; Set called\n", err)
})

6. 测试

Set a:b
Get a

[root@VM-24-14-centos src]# curl 'http://127.0.0.1:8090/set?value=b&key=a'
"a":"b"; <nil>; Set called
[root@VM-24-14-centos src]# curl http://127.0.0.1:8090/get?key=a
"a":"b"; <nil> Get Called

7. web分离

类似db分离，web也是单独的模块

创建
处理get
处理set

type Server struct {
  db *db.Database
}

func NewServer(db *db.Database) *Server {
  return &Server{db: db}
}

func (s *Server) GetHandler(w http.ResponseWriter, r *http.Request) {
  r.ParseForm()
  key := r.Form.Get("key")
  value, err := s.db.GetKey(key)
  fmt.Fprintf(w, "%q:%q; %v Get Called\n", key, value, err)
}

func (s *Server) SetHandler(w http.ResponseWriter, r *http.Request) {
  r.ParseForm()
  key := r.Form.Get("key")
  value := r.Form.Get("value")
  err := s.db.SetKey(key, []byte(value))
  fmt.Fprintf(w, "%q:%q; %v; Set called\n", key, value, err)
}

8. 更新main

web模块在main中调用

svr := web.NewServer(db)
http.HandleFunc("/get", svr.GetHandler)
http.HandleFunc("/set", svr.SetHandler)

log.Fatal(http.ListenAndServe(*httpAddress, nil))

参考资料

本节完整代码：https://github.com/YuriyNasretdinov/distribkv/tree/part1

youtube视频：https://www.youtube.com/watch?v=oPwGrCoOUdo&list=PLWwSgbaBp9XrMkjEhmTIC37WX2JfwZp7I&index=2

B站视频：https://www.bilibili.com/video/BV1nR4y177YM?p=1