fabric
/
logbox
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

update sdk proto, upd impl starage
continuous-integration/drone/tag Build is passing
Details
continuous-integration/drone/push Build is passing
Details

rr-01-add-metrics v0.1.3
loveckiy.ivan 11 months ago
parent
89b6b60b77
commit
b6f87e0bd4
12 changed files with 130 additions and 40 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      README.md
  2. 8
      pkg/logstore/circuit_breaker.go
  3. 20
      pkg/logstore/model.go
  4. 6
      pkg/logstore/storage.go
  5. 56
      pkg/model/sdk/model.pb.go
  6. 5
      pkg/model/sdk/model.proto
  7. 6
      pkg/service/builder.go
  8. 3
      pkg/service/interface.go
  9. 20
      pkg/service/model.go
  10. 3
      pkg/transport/grpcserver/methods/search.go
  11. 35
      pkg/transport/grpcserver/methods/upsert.go
  12. 3
      script/postgres.sql

5
README.md
View File

@ -66,3 +66,8 @@
//Trace: - логировать обработки запросов
### Генерация протофайла
1. Переходим в директорию ``pkg/model/sdk``
2. Выполняем в терминале
``protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative ./model.proto``
3. Cгенерированные файлы находятся в директории ``pkg/model/sdk``

8
pkg/logstore/circuit_breaker.go
View File

@ -21,7 +21,7 @@ type StorageCB interface {
Unload(ctx context.Context, in SearchIn) (out []byte, err error)
Count(ctx context.Context) (count int, err error)
NewEvent(ctx context.Context, uid, config, level, msg, name, srv, time string, timeunix int64) (result Log)
NewEvent(ctx context.Context, uid, config, level, msg, name, srv string, t time.Time, timeunix int64) (result Log)
NewSearch(columns map[string]string, search string, draw int, limit int, skip int, orderBy int, dir string) (result SearchIn)
}
@ -122,8 +122,8 @@ func (s *storageImplCB) Count(ctx context.Context) (count int, err error) {
return count, nil
}
// New - фабрика, возвращаем объект
func (s *storageImplCB) NewEvent(ctx context.Context, uid, config, level, msg, name, srv, time string, timeunix int64) (result Log) {
// NewEvent - фабрика, возвращаем объект
func (s *storageImplCB) NewEvent(ctx context.Context, uid, config, level, msg, name, srv string, t time.Time, timeunix int64) (result Log) {
return Log{
Uid: uid,
Config: config,
@ -131,7 +131,7 @@ func (s *storageImplCB) NewEvent(ctx context.Context, uid, config, level, msg, n
Msg: msg,
Name: name,
Srv: srv,
Time: time,
Time: t,
TimeUnix: timeunix,
}
}

20
pkg/logstore/model.go
View File

@ -1,15 +1,17 @@
package logstore
import "time"
type Log struct {
DTRowID string `json:"DT_RowId"`
Uid string `json:"uid"`
Config string `json:"config"`
Level string `json:"level"`
Msg string `json:"msg"`
Name string `json:"name"`
Srv string `json:"srv"`
Time string `json:"time"`
TimeUnix int64 `json:"timeunix"`
DTRowID string `json:"DT_RowId"`
Uid string `json:"uid"`
Config string `json:"config"`
Level string `json:"level"`
Msg string `json:"msg"`
Name string `json:"name"`
Srv string `json:"srv"`
Time time.Time `json:"time"`
TimeUnix int64 `json:"timeunix"`
}
type SearchOut struct {

6
pkg/logstore/storage.go
View File

@ -22,7 +22,7 @@ type Storage interface {
Unload(ctx context.Context, in SearchIn) (out []byte, err error)
Count(ctx context.Context) (count int, err error)
NewEvent(ctx context.Context, uid, config, level, msg, name, srv, time string, timeunix int64) (result Log)
NewEvent(ctx context.Context, uid, config, level, msg, name, srv string, t time.Time, timeunix int64) (result Log)
NewSearch(columns map[string]string, search string, draw int, limit int, skip int, orderBy int, dir string) (result SearchIn)
}
@ -239,7 +239,7 @@ func (s *storageImpl) Count(ctx context.Context) (count int, err error) {
}
// NewEvent - фабрика, возвращаем объект
func (s *storageImpl) NewEvent(ctx context.Context, uid, config, level, msg, name, srv, time string, timeunix int64) (result Log) {
func (s *storageImpl) NewEvent(ctx context.Context, uid, config, level, msg, name, srv string, t time.Time, timeunix int64) (result Log) {
return Log{
Uid: uid,
Config: config,
@ -247,7 +247,7 @@ func (s *storageImpl) NewEvent(ctx context.Context, uid, config, level, msg, nam
Msg: msg,
Name: name,
Srv: srv,
Time: time,
Time: t,
TimeUnix: timeunix,
}
}

56
pkg/model/sdk/model.pb.go
View File

@ -25,13 +25,17 @@ type Event struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Config string `protobuf:"bytes,1,opt,name=config,proto3" json:"config,omitempty"`
Level string `protobuf:"bytes,2,opt,name=level,proto3" json:"level,omitempty"`
Msg string `protobuf:"bytes,3,opt,name=msg,proto3" json:"msg,omitempty"`
Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"`
Srv string `protobuf:"bytes,5,opt,name=srv,proto3" json:"srv,omitempty"`
Time string `protobuf:"bytes,6,opt,name=time,proto3" json:"time,omitempty"`
Uid string `protobuf:"bytes,7,opt,name=uid,proto3" json:"uid,omitempty"`
Config string `protobuf:"bytes,1,opt,name=config,proto3" json:"config,omitempty"`
Level string `protobuf:"bytes,2,opt,name=level,proto3" json:"level,omitempty"`
Msg string `protobuf:"bytes,3,opt,name=msg,proto3" json:"msg,omitempty"`
Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"`
Srv string `protobuf:"bytes,5,opt,name=srv,proto3" json:"srv,omitempty"`
Time string `protobuf:"bytes,6,opt,name=time,proto3" json:"time,omitempty"`
Uid string `protobuf:"bytes,7,opt,name=uid,proto3" json:"uid,omitempty"`
Type string `protobuf:"bytes,8,opt,name=type,proto3" json:"type,omitempty"`
Timing string `protobuf:"bytes,9,opt,name=timing,proto3" json:"timing,omitempty"`
Request string `protobuf:"bytes,10,opt,name=request,proto3" json:"request,omitempty"`
Payload string `protobuf:"bytes,11,opt,name=payload,proto3" json:"payload,omitempty"`
}
func (x *Event) Reset() {
@ -115,6 +119,34 @@ func (x *Event) GetUid() string {
return ""
}
func (x *Event) GetType() string {
if x != nil {
return x.Type
}
return ""
}
func (x *Event) GetTiming() string {
if x != nil {
return x.Timing
}
return ""
}
func (x *Event) GetRequest() string {
if x != nil {
return x.Request
}
return ""
}
func (x *Event) GetPayload() string {
if x != nil {
return x.Payload
}
return ""
}
//TODO: Если делать поиск с возможностью учитывать регист создать структуру Search (используется в корне и колонках)
type Column struct {
state protoimpl.MessageState
@ -459,7 +491,7 @@ var File_model_proto protoreflect.FileDescriptor
var file_model_proto_rawDesc = []byte{
0x0a, 0x0b, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x6c,
0x6f, 0x67, 0x62, 0x6f, 0x78, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x22, 0x93, 0x01, 0x0a,
0x6f, 0x67, 0x62, 0x6f, 0x78, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x22, 0xf3, 0x01, 0x0a,
0x05, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x14,
0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c,
@ -469,7 +501,13 @@ var file_model_proto_rawDesc = []byte{
0x76, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x72, 0x76, 0x12, 0x12, 0x0a, 0x04,
0x74, 0x69, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x69, 0x6d, 0x65,
0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75,
0x69, 0x64, 0x22, 0x3e, 0x0a, 0x06, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x12, 0x0a, 0x04,
0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09,
0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67,
0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x12, 0x18,
0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52,
0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c,
0x6f, 0x61, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f,
0x61, 0x64, 0x22, 0x3e, 0x0a, 0x06, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x12, 0x0a, 0x04,
0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65,
0x12, 0x20, 0x0a, 0x0b, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x56, 0x61, 0x6c,

5
pkg/model/sdk/model.proto
View File

@ -12,6 +12,11 @@ message Event {
string srv = 5;
string time = 6;
string uid = 7;
string type = 8;
string timing = 9;
string request = 10;
string payload = 11;
}
//TODO: Если делать поиск с возможностью учитывать регист создать структуру Search (используется в корне и колонках)

6
pkg/service/builder.go
View File

@ -1,6 +1,8 @@
package service
func (s *builder) NewUpsertIn(uid, config, level, msg, name, srv, time string, timeunix int64) (result UpsertIn) {
import "time"
func (s *builder) NewUpsertIn(uid, config, level, msg, name, srv string, t time.Time, timeunix int64) (result UpsertIn) {
log := Log{
Uid: uid,
Config: config,
@ -8,7 +10,7 @@ func (s *builder) NewUpsertIn(uid, config, level, msg, name, srv, time string, t
Msg: msg,
Name: name,
Srv: srv,
Time: time,
Time: t,
TimeUnix: timeunix,
}

3
pkg/service/interface.go
View File

@ -2,6 +2,7 @@ package service
import (
"context"
"time"
"git.lowcodeplatform.net/fabric/lib"
"git.lowcodeplatform.net/fabric/logbox/pkg/logstore"
@ -21,7 +22,7 @@ type builder struct {
}
type Builder interface {
NewUpsertIn(uid, config, level, msg, name, srv, time string, timeunix int64) (result UpsertIn)
NewUpsertIn(uid, config, level, msg, name, srv string, t time.Time, timeunix int64) (result UpsertIn)
NewSearchIn(columns map[string]string, search string, draw int, limit int, skip int, orderBy int, dir string) (result SearchIn)
}

20
pkg/service/model.go
View File

@ -1,15 +1,17 @@
package service
import "time"
type Log struct {
DTRowID string `json:"DT_RowId"`
Uid string `json:"uid"`
Config string `json:"config"`
Level string `json:"level"`
Msg string `json:"msg"`
Name string `json:"name"`
Srv string `json:"srv"`
Time string `json:"time"`
TimeUnix int64 `json:"timeunix"`
DTRowID string `json:"DT_RowId"`
Uid string `json:"uid"`
Config string `json:"config"`
Level string `json:"level"`
Msg string `json:"msg"`
Name string `json:"name"`
Srv string `json:"srv"`
Time time.Time `json:"time"`
TimeUnix int64 `json:"timeunix"`
}
type StatIn struct {

3
pkg/transport/grpcserver/methods/search.go
View File

@ -52,10 +52,11 @@ func (s *Methods) Search(ctx context.Context, in *pb.SearchRequest) (out *pb.Sea
data := []*pb.Event{}
for _, v := range res.Data {
t := v.Time.String()
data = append(data, &pb.Event{
Config: v.Config,
Level: v.Level,
Time: v.Time,
Time: t,
Msg: v.Msg,
Uid: v.Uid,
Srv: v.Srv,

35
pkg/transport/grpcserver/methods/upsert.go
View File

@ -3,6 +3,8 @@ package methods
import (
"context"
"fmt"
"strconv"
"strings"
"time"
pb "git.lowcodeplatform.net/fabric/logbox/pkg/model/sdk"
@ -26,15 +28,22 @@ func (s *Methods) Upsert(ctx context.Context, in *pb.UpsertRequest) (out *pb.Ups
}
for _, v := range in.Events {
// можно отправлять разные типы даты (для обратной совместимости)
// 1 - нормальная дата (без наносекунд)
timeUnix, err = time.Parse(time.RFC3339, v.Time)
if err != nil {
timeUnix, err = convertNanoInt64ToTime(v.Time)
}
if err != nil {
err = fmt.Errorf("error. convert data to timeInix failed (expected format: %s). value: %s, err: %s", time.RFC3339, v.Time, err)
return out, nil
}
record := s.src.NewUpsertIn(v.Uid, v.Config, v.Level, v.Msg, v.Name, v.Srv, v.Time, timeUnix.Unix())
record := s.src.NewUpsertIn(v.Uid, v.Config, v.Level, v.Msg, v.Name, v.Srv, timeUnix, timeUnix.UnixMicro())
_, err = s.src.Upsert(ctx, record)
if err != nil {
fmt.Printf("timeUnix: %s, obj: %+v\n err: %s\n", timeUnix, record, err)
err = fmt.Errorf("error upsert. err: %s", err)
return out, nil
}
@ -42,3 +51,27 @@ func (s *Methods) Upsert(ctx context.Context, in *pb.UpsertRequest) (out *pb.Ups
return out, nil
}
// convertNanoInt64ToTime конвертируем строку типа 1687600017.636205
// (можно без точки + дополняет нулями до полного размера) в формат даты
func convertNanoInt64ToTime(s string) (result time.Time, err error) {
if len(s) < 10 {
err = fmt.Errorf("incorrect fotmat convert %s to time (exp: 1687599886.7690198, or without point)", s)
return
}
s = strings.Replace(s, ".", "", 1)
for i := len(s); i < 16; i++ {
s = s + "0"
}
s = s[:16]
i, err := strconv.ParseInt(s, 0, 64)
if err != nil {
err = fmt.Errorf("error convert %s to time, err: %s", s, err)
return
}
result = time.UnixMicro(i)
return result, err
}

3
script/postgres.sql
View File

@ -7,7 +7,8 @@ create table public.logs
name varchar,
srv varchar,
time timestamp,
timeunix integer not null
timeunix bigserial not null,
payload varchar
) PARTITION BY RANGE (timeunix);
create index IF NOT EXISTS status_idx on public.logs (timeunix);

Write Preview
Loading…
Cancel
Save