怎么实现一个分布式kv系统 #1 local database

• • • 摘要
    • 环境配置
    • • 1. 远程服务器[可选择]
      • 2. 配置VSCode
      • 3. 设置go mod
    • 编程实现
    • • 1. 导入boltdb
      • 2. 设置flag
      • 3. 创建HTTP API
      • 4. DB模块
      • 5. 更新API
      • 6. 测试
      • 7. web分离
      • 8. 更新main
    • 参考资料

本节要实现的有2点

• 环境配置：远程服务器 + VS Code设置
• 实现类似redis的set和get功能
  • 基于bolt.DB实现db模块
  • 实现简单的http web模块

远程服务器比较稳定，当然也是多折腾体验。

腾讯云服务器购买详情&一些奇奇怪怪的软件设置，请参考: 详细的配置信息

安装 Remote SSH插件

连接服务器

切换第一个tab，打开资源管理器，添加目录

查看go版本和配置

[root@VM-24-14-centos ~]# go version
go version go1.17 linux/amd64
[root@VM-24-14-centos ~]# echo $GOPATH
/data/go
[root@VM-24-14-centos ~]# echo $GOROOT
/usr/local/go

vscode安装tools

设置vscode go的选项

    "go.installDependenciesWhenBuilding": true,
    "go.useCodeSnippetsOnFunctionSuggestWithoutType": true,
    "go.autocompleteUnimportedPackages": true,
    "go.gotoSymbol.includeImports": true,
    "go.useCodeSnippetsOnFunctionSuggest": true,
    "go.inferGopath": true,

[root@VM-24-14-centos go]# mkdir kv-demo
[root@VM-24-14-centos go]# cd kv-demo
[root@VM-24-14-centos kv-demo]# go mod init example.com/kv-demo
[root@VM-24-14-centos kv-demo]# go mod tidy
[root@VM-24-14-centos kv-demo]# go mod vendor
[root@VM-24-14-centos kv-demo]# go run main.go

目前就一个参数db-location,后续可以添加

// main.go
var (
	dbLocation = flag.String("db-location", "my.db", "The path to the database location")
)

func main() {
	flag.Parse()
	db, err := bolt.Open(*dbLocation, 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
}

先简单的mock http api

// main.go
http.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "Get Called")
	})

http.HandleFunc("/set", func(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Set called")
})

log.Fatal(http.ListenAndServe(":8090", nil))

[root@VM-24-14-centos kv-demo]# curl http://127.0.0.1:8090/set
Set called
[root@VM-24-14-centos kv-demo]# curl http://127.0.0.1:8090/get
Get Called

目前位置db的功能都在main里面，我们分离出单独的db模块

• 创建DB
• 设置key
• 获取key
• 创建默认Bucket

func NewDatabase(dbLocation string) (db *Database, closeFunc func() error, err error) {
	boltDb, err := bolt.Open(dbLocation, 0600, nil)
	if err != nil {
		return nil, nil, err
	}
	db = &Database{db: boltDb}
	closeFunc = boltDb.Close

	if err := db.createDefaultBucket(); err != nil {
		return nil, nil, fmt.Errorf("create default bucket failed: %v", err)
	}

	return db, closeFunc, nil
}

func (db *Database) SetKey(key string, value []byte) error {
	return db.db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(defaultBucket)
		return b.Put([]byte(key), value)
	})
}

func (db *Database) GetKey(key string) ([]byte, error) {
	var rc []byte
	err := db.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(defaultBucket)
		rc = b.Get([]byte(key))
		return nil
	})
	return rc, err
}

func (db *Database) createDefaultBucket() error {
	return db.db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte(defaultBucket))
		return err
	})
}

更新set & get http api来

	http.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		key := r.Form.Get("key")
		value, err := db.GetKey(key)
		fmt.Fprintf(w, "%q:%q; %v Get Calledn", key, value, err)
	})


	http.HandleFunc("/set", func(w http.ResponseWriter, r *http.Request) {
		r.ParseForm()
		key := r.Form.Get("key")
		value := r.Form.Get("value")
		err := db.SetKey(key, []byte(value))
		fmt.Fprintf(w, "err: %v; Set calledn", err)
	})

• Set a:b
• Get a

[root@VM-24-14-centos src]# curl 'http://127.0.0.1:8090/set?value=b&key=a'
"a":"b"; ; Set called
[root@VM-24-14-centos src]# curl http://127.0.0.1:8090/get?key=a
"a":"b";  Get Called

类似db分离，web也是单独的模块

• 创建
• 处理get
• 处理set

type Server struct {
	db *db.Database
}

func NewServer(db *db.Database) *Server {
	return &Server{db: db}
}

func (s *Server) GetHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	key := r.Form.Get("key")
	value, err := s.db.GetKey(key)
	fmt.Fprintf(w, "%q:%q; %v Get Calledn", key, value, err)
}

func (s *Server) SetHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	key := r.Form.Get("key")
	value := r.Form.Get("value")
	err := s.db.SetKey(key, []byte(value))
	fmt.Fprintf(w, "%q:%q; %v; Set calledn", key, value, err)
}

web模块在main中调用

	svr := web.NewServer(db)
	http.HandleFunc("/get", svr.GetHandler)
	http.HandleFunc("/set", svr.SetHandler)

	log.Fatal(http.ListenAndServe(*httpAddress, nil))

本节完整代码：https://github.com/YuriyNasretdinov/distribkv/tree/part1

youtube视频：https://www.youtube.com/watch?v=oPwGrCoOUdo&list=PLWwSgbaBp9XrMkjEhmTIC37WX2JfwZp7I&index=5&t=746s

B站视频：https://www.bilibili.com/video/BV1jY411L7SA/