智能风控决策引擎系统代码实现篇（十）加载DSL转换为决策流DecisionFlow，启动Gin路由、决策流启动执行

在上一篇文章中，我们已经介绍了Dsl元信息，决策流执行基本要素INode接口，以及决策流构成DecisionFlow、决策流执行上下文记录器Pipeline。

本文接着介绍，将决策流yaml文件加载为Dsl结构体到内存，而后转换为决策流DecisionFlow，并启动Gin路由，监听执行请求。

本文完成后的目录结构

.
├── LICENSE
├── README.en.md
├── README.md
├── api
│   ├── engine_handler.go  # 路由注册后的相关处理handler
│   └── router.go  # gin路由注册
├── cmd
│   └── risk_engine
│       ├── config.yaml
│       └── engine.go  # Main入口，配置加载、日志初始化、决策流加载等
├── configs
│   ├── config.go # 配置文件结构体
│   └── const.go  # 常量定义
├── core
│   ├── dsl.go  # DSL定义与方法，对应yaml元信息，转换为可执行的决策流DecisionFlow
│   ├── flow.go  # DecisionFlow定义
│   ├── inode.go  # 具体执行节点接口定义
│   ├── kernel.go  # 本地缓存的所有可执行的决策流
│   └── pipeline.go  # 决策流执行时的上下文信息
├── global
│   └── config.go  # 记录配置文件对应的全局变量
├── go.mod
├── go.sum
└── internal
    └── log
        ├── defualt_logger.go  # 默认日志实现
        ├── logger.go  # 自定义的日志接口
        ├── logger_test.go
        └── out  # 日志输出文件

一、kernel(全部执行流的本地缓存器）

我们可能有许多的yaml文件，对应许多的决策流，因此我们需要一个结构来保存所有加载到内存后的决策流。

row_risk_engine/core/kernel.go

package core

import (
	"crypto/md5"
	"errors"
	"fmt"
	"github.com/liyouming/risk_engine/configs"
	"github.com/liyouming/risk_engine/internal/log"
	yaml "gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"path/filepath"
)

// Kernel 相当于本地缓存，管理着加载到本地缓存后的所有决策流
type Kernel struct {
	DecisionFlowMap map[string]*DecisionFlow
}

func NewKernel() *Kernel {
	return &Kernel{DecisionFlowMap: make(map[string]*DecisionFlow)}
}

// LoadDsl  加载DSL，并转换为决策流，存入本地缓存
func (kernel *Kernel) LoadDsl(method, path string) {
	var yamls map[string][]byte
	var err error
	if method == configs.FILE {
		yamls, err = kernel.LoadFromFile(path)
	} else {
		yamls, err = kernel.LoadFromDb()
	}
	if err != nil {
		log.Errorf("load dsl fail, method %s, path %s, err %s", method, path, err)
		return
	}
	for k, v := range yamls {
		dsl := new(Dsl)
		err := yaml.Unmarshal(v, dsl)
		if err != nil {
			log.Errorf("file %s convert dsl error: %s", k, err)
			continue
		}
		if !dsl.CheckValid() {
			log.Errorf("file %s dsl check error: %s", k, err)
			continue
		}
		flow, err := dsl.ConvertToDecisionFlow()
		key := kernel.getMapKey(dsl.Key, dsl.Version)
		if err != nil {
			log.Errorf("dsl %s convert to flow error: %s", key, err)
			continue
		}
		if _, ok := kernel.DecisionFlowMap[key]; ok {
			log.Errorf("dsl load repeat %s", key)
		}
		flow.Md5 = fmt.Sprintf("%x", md5.Sum(v))
		kernel.DecisionFlowMap[key] = flow //重复后一个覆盖前一个
	}
}

func (kernel *Kernel) LoadFromFile(path string) (yamls map[string][]byte, err error) {
	//get file list
	files := make([]string, 0)
	err = filepath.Walk(path, func(fp string, info os.FileInfo, err error) error {
		if filepath.Ext(fp) == ".yaml" {
			files = append(files, fp)
		}
		return err
	})

	//read file
	yamls = make(map[string][]byte)
	for _, file := range files {
		yamlFile, err := ioutil.ReadFile(file)
		if err != nil {
			log.Errorf("load file %s error: %s", file, err)
			continue
		}
		yamls[file] = yamlFile
	}

	if len(yamls) == 0 {
		err = errors.New("no valid dsl") //errcode
		return
	}
	return
}

func (kernel *Kernel) getMapKey(key, version string) string {
	return fmt.Sprintf("%s-%s", key, version)
}

//校验dsl yaml完整性
func (kernel *Kernel) CheckDslValid(dsl *Dsl) bool {
	return true
}

func (kernel *Kernel) LoadFromDb() (yamls map[string][]byte, err error) {
	err = errors.New("not finished")
	return
}

func (kernel *Kernel) GetAllDecisionFlow() map[string]*DecisionFlow {
	return kernel.DecisionFlowMap
}

func (kernel *Kernel) GetDecisionFlow(key, version string) (*DecisionFlow, error) {
	if flow, ok := kernel.DecisionFlowMap[kernel.getMapKey(key, version)]; ok {
		return flow, nil
	}
	return (*DecisionFlow)(nil), errcode.DslErrorNotFound
}

二、启动Gin路由

在Main函数入口，我们就应该加载相关Dsl到本地缓存，并启动路由监听，但需要封装为函数，模块化管理。

如下代码，通过api.Init()完成Dsl的加载和gin路由的初始化

row_risk_engine/cmd/risk_engine/engine.go

package main

import (
	"context"
	"flag"
	"github.com/liyouming/risk_engine/api"

	"github.com/liyouming/risk_engine/internal/log"

	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/liyouming/risk_engine/configs"
	"github.com/liyouming/risk_engine/global"
)

func main() {
	c := flag.String("c", "", "config file path")
	flag.Parse()
	conf, err := configs.LoadConfig(*c)
	if err != nil {
		panic(err) // 加载配置文件失败，直接退出,因为后续操作无意义了
	}
	global.ServerConf = &conf.Server
	global.AppConf = &conf.App

	// 初始化日志模块
	log.InitLogger(global.AppConf.LogMethod, global.AppConf.LogPath)

	// dsl加载与路由监听
	api.Init()

	//graceful restart
	quit := make(chan os.Signal)
	signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT)
	<-quit
	log.Info("shutting down server...")

	// 上面接受到退出信号后，5s后退出
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	select {
	case <-ctx.Done():
		log.Info("timeout of 5 seconds")
	}
	log.Info("server exiting")
}

dsl加载与路由注册

在此文件中，我们首先定义了kernel变量作为本地缓存，而后将指定文件夹下的yaml文件加载为Dsl，并转换为DecisionFlow，最终存于kernel中，方便后续随时取用，最后创建了EngineHandler，完成路由注册与服务启动。

risk_engine/api/router.go

package api

import (
	"fmt"

	"github.com/gin-gonic/gin"
	"github.com/liyouming/risk_engine/core"
	"github.com/liyouming/risk_engine/global"
	"github.com/liyouming/risk_engine/internal/log"
)

func Init() { //conf
	kernel := core.NewKernel()
	kernel.LoadDsl(global.AppConf.DslLoadMethod, global.AppConf.DslLoadPath)

	engineHandler := NewEngineHandler(kernel)

	router := gin.Default()
	router.POST("/engine/run", engineHandler.Run)
	router.GET("/engine/list", engineHandler.List)

	router.Run(fmt.Sprintf(":%d", global.ServerConf.Port)) //conf

	log.Infof("[HTTP] Listening on %d", global.ServerConf.Port)
}

risk_engine/api/engine_handler.go

该文件主要包括两个方法，Run和List，对应两个路由请求的handler。

• Run：完成参数绑定，然后使用service找到对应的决策流，执行。
• List：查看本地缓存的所有决策流。

package api

import (
	"net/http"

	"github.com/gin-gonic/gin"
	"github.com/liyouming/risk_engine/core"
	"github.com/liyouming/risk_engine/internal/log"
)

type EngineHandler struct {
	kernel *core.Kernel
}

func NewEngineHandler(kernel *core.Kernel) *EngineHandler {
	return &EngineHandler{kernel: kernel}
}

func (handler *EngineHandler) Run(c *gin.Context) {
	code := 200
	errs := ""
	var request dto.EngineRunRequest
	err := c.ShouldBindJSON(&request)
	if err != nil {
		code = 500
		c.JSON(http.StatusBadRequest, gin.H{
			"code":  code, //todo
			"error": err.Error(),
		})
		return
	}
	log.Infof("======[trace] request start req_id %s======", request.ReqId)
	svr := service.NewEngineService(handler.kernel)
	result, err := svr.Run(c, &request)
	if err != nil {
		code = 501
		errs = err.Error()
	}
	log.Infof("======[trace] request end req_id %s======", request.ReqId)
	c.JSON(http.StatusOK, gin.H{
		"code":   code,
		"result": result,
		"error":  errs,
	})
}

func (handler *EngineHandler) List(c *gin.Context) {
	data := make([]*dto.Dsl, 0)
	for _, flow := range handler.kernel.GetAllDecisionFlow() {
		dsl := &dto.Dsl{Key: flow.Key, Version: flow.Version, Md5: flow.Md5}
		data = append(data, dsl)
	}
	c.JSON(http.StatusOK, gin.H{
		"code":   200,
		"result": data,
		"error":  "",
	})
}

三、决策流执行

通过API接口请求，获取到对应的参数后，从本地缓存取出需要执行的策略，执行，并记录执行相关信息。

risk_engine/service/service.go

package service

import (
	"fmt"
	"github.com/liyouming/risk_engine/internal/dto"
	"github.com/liyouming/risk_engine/internal/log"
	"github.com/liyouming/risk_engine/internal/util"
	"github.com/liyouming/risk_engine/core"
	

	"time"

	"github.com/gin-gonic/gin"
	
	
)

type EngineService struct {
	startTime time.Time
	kernel    *core.Kernel
}

func NewEngineService(kernel *core.Kernel) *EngineService {
	builtinUdf()
	return &EngineService{kernel: kernel}
}

// dto.DslRunResponse
func (service *EngineService) Run(c *gin.Context, req *dto.EngineRunRequest) (*dto.EngineRunResponse, error) {
	service.startTime = time.Now()
	go func() {
		defer func() {
			if err := recover(); err != nil {
				log.Error(err)
			}
		}()
	}()
	flow, err := service.kernel.GetDecisionFlow(req.Key, req.Version)
	if err != nil {
		return (*dto.EngineRunResponse)(nil), err
	}

	ctx := core.NewPipelineContext()

	//fill feature value from request features
	features := make(map[string]core.IFeature)
	for name, feature := range flow.FeatureMap {
		if val, ok := req.Features[name]; ok { //in request params
			featureType, err := util.GetType(val) //check data
			if err != nil {                       //warning: unknow type
				log.Errorf("type check error: %s", err)
			}
			if !util.MatchType(featureType, feature.GetType().String()) {
				log.Warnf("request feature type is not match! %s", fmt.Sprintf("%s type is %s, required %s", name, core.GetFeatureType(featureType), feature.GetType()))
				continue
			}
			features[name] = feature
			if feature.GetType() == core.TypeDate {
				valDate, _ := util.ToDate(val.(string))
				features[name].SetValue(valDate)
			} else {
				features[name].SetValue(val)
			}
		} else {
			log.Warn("request lack feature: %s", name)
		}
	}
	log.Infof("======request features %v======", features)
	ctx.SetFeatures(features)
	flow.Run(ctx)

	result := ctx.GetDecisionResult()
	return service.dataAdapter(req, result), nil
}

// adapte the result and output
func (service *EngineService) dataAdapter(req *dto.EngineRunRequest, result *core.DecisionResult) *dto.EngineRunResponse {
	resp := &dto.EngineRunResponse{
		Key:       req.Key,
		ReqId:     req.ReqId,
		Uid:       req.Uid,
		StartTime: util.TimeFormat(service.startTime),
	}
	features := make([]map[string]interface{}, 0)
	for _, feature := range result.Features {
		value, ok := feature.GetValue()
		features = append(features, map[string]interface{}{"name": feature.GetName(),
			"value":     value,
			"isDefault": !ok,
		})
	}
	resp.Features = features
	tracks := make([]map[string]interface{}, 0)
	i := 1
	for _, track := range result.Tracks {
		tracks = append(tracks, map[string]interface{}{"index": i,
			"name":  track.Name,
			"label": track.Label,
		})
		i++
	}
	resp.Tracks = tracks
	hitRules := make([]map[string]interface{}, 0)
	for _, rule := range result.HitRules {
		hitRules = append(hitRules, map[string]interface{}{"id": rule.Id,
			"name":  rule.Name,
			"label": rule.Label,
		})
	}
	resp.HitRules = hitRules
	nodeResults := make([]map[string]interface{}, 0)
	for _, nodeResult := range result.NodeResults {
		if nodeResult == nil {
			continue
		}
		nodeResults = append(nodeResults, map[string]interface{}{
			"name":    nodeResult.Name,
			"id":      nodeResult.Id,
			"Kind":    nodeResult.Kind.String(),
			"tag":     nodeResult.Tag,
			"label":   nodeResult.Label,
			"IsBlock": nodeResult.IsBlock,
			"Value":   nodeResult.Value,
			"Score":   nodeResult.Score,
		})
		i++
	}
	resp.NodeResults = nodeResults
	resp.RunTime = util.TimeSince(service.startTime)
	resp.EndTime = util.TimeFormat(time.Now())
	return resp
}

func builtinUdf() {
	global.RegisterUdf("sum", udf.Sum)
}