From 293bb364c041286cb4e8527937a8860146073c76 Mon Sep 17 00:00:00 2001 From: Matias Alvin Date: Wed, 29 Jan 2020 19:14:41 +0700 Subject: [PATCH] feat(descriptor): add support for local proto descriptor Currently, grpcox depends on server reflection to get proto descriptor. It has a significant drawback, since not every grpc server support [server reflection](https://github.com/grpc/grpc/blob/master/doc/server-reflection.md#known-implementations). Using local proto files is more feasible, as every grpc server certainly have one. Even though using protofile should be simple enough, there's still a problem regarding this. Some protofile use extra plugins for their proto. i.e. gogoprotobuf is a project that does just that. The problems with plugins are most of them require explicit import to the plugin inside of the protofile. It will break grpcurl proto descriptor extraction. Thus, the plugin proto must be uploaded alongside the protofile. Also, the protofile should be modified automatically to change their import to local import. Given that, I proposed a way for the user to upload multiple protofile to grpcox. Then, use that to get the descriptor. Changelog: - Add `use local proto` checkbox in HTML client. On checked it will show upload button and list of selected proto. - `get-service` ajax will use POST when `use local proto` is checked. The uploaded protofile will be the payload for the ajax request. - Add a new route to handle POST "get-service". It will persist the uploaded protofile to `/tmp/` directory and add protos field in the resource. - Modify `openDescriptor` to use local proto if protos field in the resource is available. - Modify `openDescriptor` to return an error, as opening descriptor from local proto may fail. - Modify the main server so it can be shut down gracefully. This is necessary as grpcox need to remove persisted proto right after the server is turned off. This Pull Request will resolve #16 --- .gitignore | 3 +- core/grpcox.go | 27 +++++++ core/resource.go | 123 ++++++++++++++++++++++++++++--- core/resource_test.go | 70 ++++++++++++++++++ grpcox.go | 46 +++++++++++- handler/handler.go | 60 +++++++++++++++ handler/routes.go | 1 + index/css/proto.css | 115 +++++++++++++++++++++++++++++ index/img/file.png | Bin 0 -> 3939 bytes index/index.html | 22 +++++- index/js/proto.js | 165 ++++++++++++++++++++++++++++++++++++++++++ index/js/style.js | 27 +++++-- 12 files changed, 638 insertions(+), 21 deletions(-) create mode 100644 core/resource_test.go create mode 100644 index/css/proto.css create mode 100644 index/img/file.png create mode 100644 index/js/proto.js diff --git a/.gitignore b/.gitignore index 918573f..c2b1247 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ grpcox -log \ No newline at end of file +log +*.out \ No newline at end of file diff --git a/core/grpcox.go b/core/grpcox.go index 7c18b95..efe88e0 100644 --- a/core/grpcox.go +++ b/core/grpcox.go @@ -3,6 +3,7 @@ package core import ( "context" "os" + "reflect" "strconv" "time" @@ -31,6 +32,14 @@ type GrpCox struct { isUnixSocket func() bool } +// Proto define protofile uploaded from client +// will be used to be persisted to disk and indicator +// whether connections should reflect from server or local proto +type Proto struct { + Name string + Content []byte +} + // InitGrpCox constructor func InitGrpCox() *GrpCox { maxLife, tick := 10, 3 @@ -80,6 +89,24 @@ func (g *GrpCox) GetResource(ctx context.Context, target string, plainText, isRe return r, nil } +// GetResourceWithProto - open resource to targeted grpc server using given protofile +func (g *GrpCox) GetResourceWithProto(ctx context.Context, target string, plainText, isRestartConn bool, protos []Proto) (*Resource, error) { + r, err := g.GetResource(ctx, target, plainText, isRestartConn) + if err != nil { + return nil, err + } + + // if given protofile is equal to current, skip adding protos as it's already + // persisted in the harddisk anyway + if reflect.DeepEqual(r.protos, protos) { + return r, nil + } + + // add protos property to resource and persist it to harddisk + err = r.AddProtos(protos) + return r, err +} + // GetActiveConns - get all saved active connection func (g *GrpCox) GetActiveConns(ctx context.Context) []string { active := g.activeConn.getAllConn() diff --git a/core/resource.go b/core/resource.go index cc8be52..04819d1 100644 --- a/core/resource.go +++ b/core/resource.go @@ -5,8 +5,13 @@ import ( "context" "fmt" "io" + "io/ioutil" "log" "os" + "path/filepath" + "regexp" + "strings" + "sync" "time" "github.com/fullstorydev/grpcurl" @@ -18,22 +23,43 @@ import ( reflectpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" ) +// BasePath define path where proto file will persisted +const BasePath = "/tmp/grpcox/" + // Resource - hold 3 main function (List, Describe, and Invoke) type Resource struct { clientConn *grpc.ClientConn descSource grpcurl.DescriptorSource refClient *grpcreflect.Client + protos []Proto headers []string md metadata.MD } //openDescriptor - use it to reflect server descriptor -func (r *Resource) openDescriptor() { +func (r *Resource) openDescriptor() error { ctx := context.Background() refCtx := metadata.NewOutgoingContext(ctx, r.md) r.refClient = grpcreflect.NewClient(refCtx, reflectpb.NewServerReflectionClient(r.clientConn)) - r.descSource = grpcurl.DescriptorSourceFromServer(ctx, r.refClient) + + // if no protos available use server reflection + if r.protos == nil { + r.descSource = grpcurl.DescriptorSourceFromServer(ctx, r.refClient) + return nil + } + + protoPath := filepath.Join(BasePath, r.clientConn.Target()) + + // make list of protos name to be used as descriptor + protos := make([]string, 0, len(r.protos)) + for _, proto := range r.protos { + protos = append(protos, proto.Name) + } + + var err error + r.descSource, err = grpcurl.DescriptorSourceFromProtoFiles([]string{protoPath}, protos...) + return err } //closeDescriptor - please ensure to always close after open in the same flow @@ -50,7 +76,7 @@ func (r *Resource) closeDescriptor() { case <-done: return case <-time.After(3 * time.Second): - log.Printf("Reflection %s falied to close\n", r.clientConn.Target()) + log.Printf("Reflection %s failed to close\n", r.clientConn.Target()) return } } @@ -59,7 +85,10 @@ func (r *Resource) closeDescriptor() { // symbol can be "" to list all available services // symbol also can be service name to list all available method func (r *Resource) List(symbol string) ([]string, error) { - r.openDescriptor() + err := r.openDescriptor() + if err != nil { + return nil, err + } defer r.closeDescriptor() var result []string @@ -97,7 +126,10 @@ func (r *Resource) List(symbol string) ([]string, error) { // It also prints a description of that symbol, in the form of snippets of proto source. // It won't necessarily be the original source that defined the element, but it will be equivalent. func (r *Resource) Describe(symbol string) (string, string, error) { - r.openDescriptor() + err := r.openDescriptor() + if err != nil { + return "", "", err + } defer r.closeDescriptor() var result, template string @@ -153,7 +185,10 @@ func (r *Resource) Describe(symbol string) (string, string, error) { // Invoke - invoking gRPC function func (r *Resource) Invoke(ctx context.Context, symbol string, in io.Reader) (string, time.Duration, error) { - r.openDescriptor() + err := r.openDescriptor() + if err != nil { + return "", 0, err + } defer r.closeDescriptor() // because of grpcurl directly fmt.Printf on their invoke function @@ -202,20 +237,37 @@ func (r *Resource) Invoke(ctx context.Context, symbol string, in io.Reader) (str // Close - to close all resources that was opened before func (r *Resource) Close() { - done := make(chan int) + var wg sync.WaitGroup + + wg.Add(1) go func() { + defer wg.Done() if r.clientConn != nil { r.clientConn.Close() r.clientConn = nil } - done <- 1 + }() + + wg.Add(1) + go func() { + defer wg.Done() + err := os.RemoveAll(BasePath) + if err != nil { + log.Printf("error removing proto dir from tmp: %s", err.Error()) + } + }() + + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() }() select { - case <-done: + case <-c: return case <-time.After(3 * time.Second): - log.Printf("Connection %s falied to close\n", r.clientConn.Target()) + log.Printf("Connection %s failed to close\n", r.clientConn.Target()) return } } @@ -229,3 +281,54 @@ func (r *Resource) exit(code int) { r.Close() os.Exit(code) } + +// AddProtos to resource properties and harddisk +// added protos will be persisted in `basepath + connection target` +// i.e. connection target == 127.0.0.1:8888 +// proto files will be persisted in /tmp/grpcox/127.0.0.1:8888 +// if the directory is already there, remove it first +func (r *Resource) AddProtos(protos []Proto) error { + protoPath := filepath.Join(BasePath, r.clientConn.Target()) + err := os.MkdirAll(protoPath, 0777) + if os.IsExist(err) { + os.RemoveAll(protoPath) + err = os.MkdirAll(protoPath, 0777) + } else if err != nil { + return err + } + + for _, proto := range protos { + err := ioutil.WriteFile(filepath.Join(protoPath, "/", proto.Name), + prepareImport(proto.Content), + 0777) + if err != nil { + return err + } + } + + r.protos = protos + return nil +} + +// prepareImport transforming proto import into local path +// with exception to google proto import as it won't cause any problem +func prepareImport(proto []byte) []byte { + const pattern = `import ".+` + result := string(proto) + + re := regexp.MustCompile(pattern) + matchs := re.FindAllString(result, -1) + for _, match := range matchs { + if strings.Contains(match, "\"google/") { + continue + } + name := strings.Split(match, "/") + if len(name) < 2 { + continue + } + importString := `import "` + name[len(name)-1] + result = strings.Replace(result, match, importString, -1) + } + + return []byte(result) +} diff --git a/core/resource_test.go b/core/resource_test.go new file mode 100644 index 0000000..b3764b3 --- /dev/null +++ b/core/resource_test.go @@ -0,0 +1,70 @@ +package core + +import ( + "reflect" + "testing" +) + +func Test_prepareImport(t *testing.T) { + type args struct { + proto []byte + } + tests := []struct { + name string + args args + want []byte + }{ + { + name: "sucess change import path to local", + args: args{ + proto: []byte(` + package testing; + + import "test.com/owner/repo/content.proto";`), + }, + want: []byte(` + package testing; + + import "content.proto";`), + }, + { + name: "sucess keep google import", + args: args{ + proto: []byte(` + package testing; + + import "google/proto/buf"; + import "test.com/owner/repo/content.proto";`), + }, + want: []byte(` + package testing; + + import "google/proto/buf"; + import "content.proto";`), + }, + { + name: "sucess keep local import", + args: args{ + proto: []byte(` + package testing; + + import "repo.proto"; + import "test.com/owner/repo/content.proto";`), + }, + want: []byte(` + package testing; + + import "repo.proto"; + import "content.proto";`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := prepareImport(tt.args.proto); !reflect.DeepEqual(got, tt.want) { + t.Errorf("prepareImport() = %v, want %v", + string(got), + string(tt.want)) + } + }) + } +} diff --git a/grpcox.go b/grpcox.go index f2fe76f..e10204a 100644 --- a/grpcox.go +++ b/grpcox.go @@ -1,13 +1,16 @@ package main import ( + "context" "fmt" "log" "net/http" "os" + "os/signal" "time" "github.com/gorilla/mux" + "github.com/gusaul/grpcox/core" "github.com/gusaul/grpcox/handler" ) @@ -25,13 +28,48 @@ func main() { port := ":6969" muxRouter := mux.NewRouter() handler.Init(muxRouter) + var wait time.Duration = time.Second * 15 + srv := &http.Server{ - Handler: muxRouter, Addr: "0.0.0.0" + port, - WriteTimeout: 15 * time.Second, - ReadTimeout: 15 * time.Second, + WriteTimeout: time.Second * 15, + ReadTimeout: time.Second * 15, + IdleTimeout: time.Second * 60, + Handler: muxRouter, } fmt.Println("Service started on", port) - log.Fatal(srv.ListenAndServe()) + go func() { + log.Fatal(srv.ListenAndServe()) + }() + + c := make(chan os.Signal, 1) + + // We'll accept graceful shutdowns when quit via SIGINT (Ctrl+C) + // SIGKILL, SIGQUIT or SIGTERM (Ctrl+/) will not be caught. + signal.Notify(c, os.Interrupt) + + // Block until we receive our signal. + <-c + + // Create a deadline to wait for. + ctx, cancel := context.WithTimeout(context.Background(), wait) + defer cancel() + + err = removeProtos() + if err != nil { + log.Printf("error while removing protos: %s", err.Error()) + } + + srv.Shutdown(ctx) + log.Println("shutting down") + os.Exit(0) +} + +// removeProtos will remove all uploaded proto file +// this function will be called as the server shutdown gracefully +func removeProtos() error { + log.Println("removing proto dir from /tmp") + err := os.RemoveAll(core.BasePath) + return err } diff --git a/handler/handler.go b/handler/handler.go index 9ec5994..373e269 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "io/ioutil" "net/http" "strconv" "strings" @@ -85,6 +86,65 @@ func (h *Handler) getLists(w http.ResponseWriter, r *http.Request) { response(w, result) } +// getListsWithProto handling client request for service list with proto +func (h *Handler) getListsWithProto(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + host := vars["host"] + if host == "" { + writeError(w, fmt.Errorf("Invalid Host")) + return + } + + service := vars["serv_name"] + + useTLS, _ := strconv.ParseBool(r.Header.Get("use_tls")) + restart, _ := strconv.ParseBool(r.FormValue("restart")) + + // limit upload file to 5mb + err := r.ParseMultipartForm(5 << 20) + if err != nil { + writeError(w, err) + return + } + + // convert uploaded files to list of Proto struct + files := r.MultipartForm.File["protos"] + protos := make([]core.Proto, 0, len(files)) + for _, file := range files { + fileData, err := file.Open() + if err != nil { + writeError(w, err) + return + } + defer fileData.Close() + + content, err := ioutil.ReadAll(fileData) + if err != nil { + writeError(w, err) + } + + protos = append(protos, core.Proto{ + Name: file.Filename, + Content: content, + }) + } + + res, err := h.g.GetResourceWithProto(context.Background(), host, !useTLS, restart, protos) + if err != nil { + writeError(w, err) + return + } + + result, err := res.List(service) + if err != nil { + writeError(w, err) + return + } + + h.g.Extend(host) + response(w, result) +} + func (h *Handler) describeFunction(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) host := vars["host"] diff --git a/handler/routes.go b/handler/routes.go index e05a89f..4e5c27c 100644 --- a/handler/routes.go +++ b/handler/routes.go @@ -14,6 +14,7 @@ func Init(router *mux.Router) { ajaxRoute := router.PathPrefix("/server/{host}").Subrouter() ajaxRoute.HandleFunc("/services", corsHandler(h.getLists)).Methods(http.MethodGet, http.MethodOptions) + ajaxRoute.HandleFunc("/services", corsHandler(h.getListsWithProto)).Methods(http.MethodPost) ajaxRoute.HandleFunc("/service/{serv_name}/functions", corsHandler(h.getLists)).Methods(http.MethodGet, http.MethodOptions) ajaxRoute.HandleFunc("/function/{func_name}/describe", corsHandler(h.describeFunction)).Methods(http.MethodGet, http.MethodOptions) ajaxRoute.HandleFunc("/function/{func_name}/invoke", corsHandler(h.invokeFunction)).Methods(http.MethodPost, http.MethodOptions) diff --git a/index/css/proto.css b/index/css/proto.css new file mode 100644 index 0000000..6e2d15c --- /dev/null +++ b/index/css/proto.css @@ -0,0 +1,115 @@ +#proto-input { + margin-top: 10px; + margin-bottom: 15px; +} + +[type="file"] { + border: 0; + clip: rect(0, 0, 0, 0); + height: 1px; + overflow: hidden; + padding: 0; + position: absolute !important; + white-space: nowrap; + width: 1px; +} +[type="file"] + label { + background-color: #59698d; + border-radius: 10px; + color: #fff; + cursor: pointer; + display: inline-block; + font-family: 'Poppins', sans-serif; + font-size: 1rem; + font-weight: 600; + padding: 5px; + transition: background-color 0.3s; +} + +[type="file"]:focus + label, +[type="file"] + label:hover { + background-color: #324674; +} + +[type="file"]:focus + label { + outline: 1px dotted #000; + outline: -webkit-focus-ring-color auto 5px; +} + +.proto-top-collection { + display: inline-flex; + flex-direction: row; + align-items: center; + justify-content: space-between; + width: 100%; +} + +.proto-toggle { + color: rgba(0, 0, 0, 0.5); + font-size: 0.8rem; +} + +.proto-toggle:hover { + cursor: pointer; +} + +.proto-collection { + background-color: #e0dfe6; + border-radius: 5px; + min-height: 120px; + width: 100%; + display: inline-flex; + align-items: center; + align-content: center; + flex-direction: row; + padding: 20px; + padding-right: 0px; + flex-wrap: wrap; +} + +.proto-item { + height: 80px; + margin-right: 20px; + margin-left: 10px; + margin-bottom: 30px; + + display: inline-flex; + flex-direction: row; + align-items: flex-start; +} + +.proto-icon { + height: 80px; + opacity: 0.5; +} + +.proto-desc { + display: inline-flex; + flex-direction: column; + align-items: flex-start; + height: 100%; +} + +.proto-caption { + font-weight: 600; + color: rgba(0,0,0,0.6); + font-size: 1rem; +} + +.proto-size { + font-size: 1rem; + color: rgba(0,0,0,0.6); +} + +.proto-remove { + background-color: rgba(0,0,0,0.5); + padding: 3px !important; + margin-left: 0px !important; + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; +} + +.proto-remove:hover { + background-color: rgba(255,0,0,0.4); +} diff --git a/index/img/file.png b/index/img/file.png new file mode 100644 index 0000000000000000000000000000000000000000..131f2f69cdcf9030ee74b81971b9385edb5ac93c GIT binary patch literal 3939 zcmd5n4~xAi0Nl*3-SVixr*;u$NC_#-q(B|&Da{0Qwo%-JsB(cKq+kfyoT_${bdEO20jN_H7 zxmL$epPdzqHLH$Ts?E^zRp)se_sf%ZWH_xn6td#-4)n-2$1x|(uWbv7`RveVV~#ei zJw9#Lr5&MN_s>;g`GJ0=ee|Al`c8gPb^B?4r68*PDf^M}5ntb|kzGQ$cZg_E1RNpq10$Vxj&zgIFXkRSaeIdADlf0kF*E(j-on* zhAm}c$uKQfbq5+w;K2-#EWdQj)OZg`>td@!SA*yeERg+OXHWxn$19I_6DE>GX zUygVqj43n0)*PI}1V2JK6ZGfc=@#HyuFA&@gfn|x&A@hMZ?hTL$LxJ?297g(51N6~ z%yZ=^zUfr^$u7~`SynJeu>wKytuGi8!NsG_-hgXe0BCqGY6eg^IyZYE4FT`xdQj4p zMag+?P)51Up`^=fD9tB*2P4umd~u+1V2sENE_~0y;UwcC_9H0R5Wbg^5y?>2T%;zC zPpEaxOF!@VL;B5@$gf z%0@Gx;b~kEGz8r9f@{e?Er8~QpZ8HIcGW%tn#MDsz(2cyqe%;I`6sZl=?wdF(N+9p-*Uk-l`?C3{$5ouZmUfZ>AYrzsf=vA(E8M!wJLvb9&ybA0oUpyr+T| zY;FLZM&mt>AVyW2dfz7wX%CZ`1`Ekgjx>a&t03wGTa3rPVJo^LvkBYl8>d;$1UC10PQXxXF+PJ@{C z34H!;uHM2XtQ2zV42p4^WwEb!EUH1VI|*Y&550I-gd_Sp8uG4z@k4E~VW3d8fT5BL zQ_TN5DepqDjxKEpw7hOIX$%bilVF^(;XoqIuHd=Ux^-CNkT)A{D$&Ne*tGX{b>&32D2ZLUw3~g{C21L6IitKt+DyFltR55wJxV()2u5d2~6|{GxLBqi-uzexQ>ED z41ikwpA9vzqxqj*7Zo+6w}R2v4>=4w~%65_SNKE8Pr`M^{uI%uP} z^+?D96$KmZBt~`hwP8MgcPhW`ic}V=O!vi^7iV=ss2U2$%j|^Di(aT!d_=y?MY*~>At3Q~G)L2cq`$n+B zc%;k{w=Zq2H!w2$ACQd(MyMkMgw9rVtX?5jtz)QimMG|~^IZx$Ro6F0Q@jyf{jye3 zA-(~Pa-+~3V?G}knO0YB+5(M#6$u0%au&&oAX3aGsEZAZhmN4)?SZ;-Tb=FoPqNf4 ztx5&Q|1`uv1^K%v-C&{nwWmEv+YXiaL$vtEhsjH#l*qL`&(dbj-y`2ut|fc~KZ@_D z8LGA{H7Q6nnstoN`Y4P20fHUf2HO0;8b$je*Yefx4E7Q|HXIrm)~?oTd`j*+owEJ@vW9@)>Mz zhwNr%%bEm1kdIUL>@b4WOfU+Pyg)N*%7X@h`G{I>fOB#U5+8`8H6U$9*)mu{uC#)# zbfy3x&TWpfu$4qTFQkB^dq{o{fz$hiHWa%1fep24cWh8v5he`E^MIW9BX1}!)~nfv zZ4?dNYkR`XlUpdgo*Nm=Z(UEGRy2smbR`~4Es>U26Q9EZ|9t69g0V7Hx;auNEzeb^ z%NM>sqI9D1iZZ^w^(FcdQTiW^My#aYQx$qD-7d^0UTA!HUB!x8X+==m1A~32Gl*Dx zMJnZ|WMlLLRZD|UshgC1|9l744%vn5y+bW+k4S5>Pvbz<{mna*-m{ouc5(OSc?895 zmagAqBWuy=%dWJ#K(FS}OLZaE_h+5mWDU>QjW%WQPXjHpQMe}uj9dIueJ$ktzBL7E zc62!`tQE!FhaZ?*dqoT!6r~?1-9@~k_+GE58)0$I5aB>UCH>a^W3a?xh>Sm%Earl= zI@!`r_E#2f_YDVsU^GX2OWM!-kF@o&ox~YYNJ03JvXp9sNt#afmI)}vzM?qGoApum?>V~4 zS43uK5BuqqJS4_wn-Ud zQMhs0Z|rDlcob6fE)0xfA{BE|2sMxmfEch=O%a zzPdRiUCE&?>z7rc=wS}jC5F|>cRd@qb~@?A0GUo=#&IdV+{lO^nU=yf)xV3hv1B!Q z&GX^-<#GL(X5GP!>F@-KfTVA2-49|!_sK_F?0+!(fBFeQ-4W*1vOq&2P;QMf;^}8L z12<`g(%E~=e||bc^S+Pk6K+UPnFs$*Iwr5;C2x&Rj`fV$6AK@}gXO`R%W|LVv3Q;P z7oH2;JwNwwWwAV2tc&(}@&5=&*u8ao+`fMwaC866Utz#hDnsb*xMX4Uo>(9h3f;Ev e+L;s+ofzx3dk;U~;57|80pC@DE2S&2Lw^GtOVM!v literal 0 HcmV?d00001 diff --git a/index/index.html b/index/index.html index 81d4ca4..cf77705 100644 --- a/index/index.html +++ b/index/index.html @@ -10,6 +10,7 @@ + @@ -28,6 +29,24 @@ +
+
+ + +
+
+ + +