diff --git a/.gitignore b/.gitignore index 918573f..c2b1247 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ grpcox -log \ No newline at end of file +log +*.out \ No newline at end of file diff --git a/core/grpcox.go b/core/grpcox.go index 7c18b95..efe88e0 100644 --- a/core/grpcox.go +++ b/core/grpcox.go @@ -3,6 +3,7 @@ package core import ( "context" "os" + "reflect" "strconv" "time" @@ -31,6 +32,14 @@ type GrpCox struct { isUnixSocket func() bool } +// Proto define protofile uploaded from client +// will be used to be persisted to disk and indicator +// whether connections should reflect from server or local proto +type Proto struct { + Name string + Content []byte +} + // InitGrpCox constructor func InitGrpCox() *GrpCox { maxLife, tick := 10, 3 @@ -80,6 +89,24 @@ func (g *GrpCox) GetResource(ctx context.Context, target string, plainText, isRe return r, nil } +// GetResourceWithProto - open resource to targeted grpc server using given protofile +func (g *GrpCox) GetResourceWithProto(ctx context.Context, target string, plainText, isRestartConn bool, protos []Proto) (*Resource, error) { + r, err := g.GetResource(ctx, target, plainText, isRestartConn) + if err != nil { + return nil, err + } + + // if given protofile is equal to current, skip adding protos as it's already + // persisted in the harddisk anyway + if reflect.DeepEqual(r.protos, protos) { + return r, nil + } + + // add protos property to resource and persist it to harddisk + err = r.AddProtos(protos) + return r, err +} + // GetActiveConns - get all saved active connection func (g *GrpCox) GetActiveConns(ctx context.Context) []string { active := g.activeConn.getAllConn() diff --git a/core/resource.go b/core/resource.go index cc8be52..04819d1 100644 --- a/core/resource.go +++ b/core/resource.go @@ -5,8 +5,13 @@ import ( "context" "fmt" "io" + "io/ioutil" "log" "os" + "path/filepath" + "regexp" + "strings" + "sync" "time" "github.com/fullstorydev/grpcurl" @@ -18,22 +23,43 @@ import ( reflectpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" ) +// BasePath define path where proto file will persisted +const BasePath = "/tmp/grpcox/" + // Resource - hold 3 main function (List, Describe, and Invoke) type Resource struct { clientConn *grpc.ClientConn descSource grpcurl.DescriptorSource refClient *grpcreflect.Client + protos []Proto headers []string md metadata.MD } //openDescriptor - use it to reflect server descriptor -func (r *Resource) openDescriptor() { +func (r *Resource) openDescriptor() error { ctx := context.Background() refCtx := metadata.NewOutgoingContext(ctx, r.md) r.refClient = grpcreflect.NewClient(refCtx, reflectpb.NewServerReflectionClient(r.clientConn)) - r.descSource = grpcurl.DescriptorSourceFromServer(ctx, r.refClient) + + // if no protos available use server reflection + if r.protos == nil { + r.descSource = grpcurl.DescriptorSourceFromServer(ctx, r.refClient) + return nil + } + + protoPath := filepath.Join(BasePath, r.clientConn.Target()) + + // make list of protos name to be used as descriptor + protos := make([]string, 0, len(r.protos)) + for _, proto := range r.protos { + protos = append(protos, proto.Name) + } + + var err error + r.descSource, err = grpcurl.DescriptorSourceFromProtoFiles([]string{protoPath}, protos...) + return err } //closeDescriptor - please ensure to always close after open in the same flow @@ -50,7 +76,7 @@ func (r *Resource) closeDescriptor() { case <-done: return case <-time.After(3 * time.Second): - log.Printf("Reflection %s falied to close\n", r.clientConn.Target()) + log.Printf("Reflection %s failed to close\n", r.clientConn.Target()) return } } @@ -59,7 +85,10 @@ func (r *Resource) closeDescriptor() { // symbol can be "" to list all available services // symbol also can be service name to list all available method func (r *Resource) List(symbol string) ([]string, error) { - r.openDescriptor() + err := r.openDescriptor() + if err != nil { + return nil, err + } defer r.closeDescriptor() var result []string @@ -97,7 +126,10 @@ func (r *Resource) List(symbol string) ([]string, error) { // It also prints a description of that symbol, in the form of snippets of proto source. // It won't necessarily be the original source that defined the element, but it will be equivalent. func (r *Resource) Describe(symbol string) (string, string, error) { - r.openDescriptor() + err := r.openDescriptor() + if err != nil { + return "", "", err + } defer r.closeDescriptor() var result, template string @@ -153,7 +185,10 @@ func (r *Resource) Describe(symbol string) (string, string, error) { // Invoke - invoking gRPC function func (r *Resource) Invoke(ctx context.Context, symbol string, in io.Reader) (string, time.Duration, error) { - r.openDescriptor() + err := r.openDescriptor() + if err != nil { + return "", 0, err + } defer r.closeDescriptor() // because of grpcurl directly fmt.Printf on their invoke function @@ -202,20 +237,37 @@ func (r *Resource) Invoke(ctx context.Context, symbol string, in io.Reader) (str // Close - to close all resources that was opened before func (r *Resource) Close() { - done := make(chan int) + var wg sync.WaitGroup + + wg.Add(1) go func() { + defer wg.Done() if r.clientConn != nil { r.clientConn.Close() r.clientConn = nil } - done <- 1 + }() + + wg.Add(1) + go func() { + defer wg.Done() + err := os.RemoveAll(BasePath) + if err != nil { + log.Printf("error removing proto dir from tmp: %s", err.Error()) + } + }() + + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() }() select { - case <-done: + case <-c: return case <-time.After(3 * time.Second): - log.Printf("Connection %s falied to close\n", r.clientConn.Target()) + log.Printf("Connection %s failed to close\n", r.clientConn.Target()) return } } @@ -229,3 +281,54 @@ func (r *Resource) exit(code int) { r.Close() os.Exit(code) } + +// AddProtos to resource properties and harddisk +// added protos will be persisted in `basepath + connection target` +// i.e. connection target == 127.0.0.1:8888 +// proto files will be persisted in /tmp/grpcox/127.0.0.1:8888 +// if the directory is already there, remove it first +func (r *Resource) AddProtos(protos []Proto) error { + protoPath := filepath.Join(BasePath, r.clientConn.Target()) + err := os.MkdirAll(protoPath, 0777) + if os.IsExist(err) { + os.RemoveAll(protoPath) + err = os.MkdirAll(protoPath, 0777) + } else if err != nil { + return err + } + + for _, proto := range protos { + err := ioutil.WriteFile(filepath.Join(protoPath, "/", proto.Name), + prepareImport(proto.Content), + 0777) + if err != nil { + return err + } + } + + r.protos = protos + return nil +} + +// prepareImport transforming proto import into local path +// with exception to google proto import as it won't cause any problem +func prepareImport(proto []byte) []byte { + const pattern = `import ".+` + result := string(proto) + + re := regexp.MustCompile(pattern) + matchs := re.FindAllString(result, -1) + for _, match := range matchs { + if strings.Contains(match, "\"google/") { + continue + } + name := strings.Split(match, "/") + if len(name) < 2 { + continue + } + importString := `import "` + name[len(name)-1] + result = strings.Replace(result, match, importString, -1) + } + + return []byte(result) +} diff --git a/core/resource_test.go b/core/resource_test.go new file mode 100644 index 0000000..b3764b3 --- /dev/null +++ b/core/resource_test.go @@ -0,0 +1,70 @@ +package core + +import ( + "reflect" + "testing" +) + +func Test_prepareImport(t *testing.T) { + type args struct { + proto []byte + } + tests := []struct { + name string + args args + want []byte + }{ + { + name: "sucess change import path to local", + args: args{ + proto: []byte(` + package testing; + + import "test.com/owner/repo/content.proto";`), + }, + want: []byte(` + package testing; + + import "content.proto";`), + }, + { + name: "sucess keep google import", + args: args{ + proto: []byte(` + package testing; + + import "google/proto/buf"; + import "test.com/owner/repo/content.proto";`), + }, + want: []byte(` + package testing; + + import "google/proto/buf"; + import "content.proto";`), + }, + { + name: "sucess keep local import", + args: args{ + proto: []byte(` + package testing; + + import "repo.proto"; + import "test.com/owner/repo/content.proto";`), + }, + want: []byte(` + package testing; + + import "repo.proto"; + import "content.proto";`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := prepareImport(tt.args.proto); !reflect.DeepEqual(got, tt.want) { + t.Errorf("prepareImport() = %v, want %v", + string(got), + string(tt.want)) + } + }) + } +} diff --git a/grpcox.go b/grpcox.go index f2fe76f..e10204a 100644 --- a/grpcox.go +++ b/grpcox.go @@ -1,13 +1,16 @@ package main import ( + "context" "fmt" "log" "net/http" "os" + "os/signal" "time" "github.com/gorilla/mux" + "github.com/gusaul/grpcox/core" "github.com/gusaul/grpcox/handler" ) @@ -25,13 +28,48 @@ func main() { port := ":6969" muxRouter := mux.NewRouter() handler.Init(muxRouter) + var wait time.Duration = time.Second * 15 + srv := &http.Server{ - Handler: muxRouter, Addr: "0.0.0.0" + port, - WriteTimeout: 15 * time.Second, - ReadTimeout: 15 * time.Second, + WriteTimeout: time.Second * 15, + ReadTimeout: time.Second * 15, + IdleTimeout: time.Second * 60, + Handler: muxRouter, } fmt.Println("Service started on", port) - log.Fatal(srv.ListenAndServe()) + go func() { + log.Fatal(srv.ListenAndServe()) + }() + + c := make(chan os.Signal, 1) + + // We'll accept graceful shutdowns when quit via SIGINT (Ctrl+C) + // SIGKILL, SIGQUIT or SIGTERM (Ctrl+/) will not be caught. + signal.Notify(c, os.Interrupt) + + // Block until we receive our signal. + <-c + + // Create a deadline to wait for. + ctx, cancel := context.WithTimeout(context.Background(), wait) + defer cancel() + + err = removeProtos() + if err != nil { + log.Printf("error while removing protos: %s", err.Error()) + } + + srv.Shutdown(ctx) + log.Println("shutting down") + os.Exit(0) +} + +// removeProtos will remove all uploaded proto file +// this function will be called as the server shutdown gracefully +func removeProtos() error { + log.Println("removing proto dir from /tmp") + err := os.RemoveAll(core.BasePath) + return err } diff --git a/handler/handler.go b/handler/handler.go index 9ec5994..373e269 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "io/ioutil" "net/http" "strconv" "strings" @@ -85,6 +86,65 @@ func (h *Handler) getLists(w http.ResponseWriter, r *http.Request) { response(w, result) } +// getListsWithProto handling client request for service list with proto +func (h *Handler) getListsWithProto(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + host := vars["host"] + if host == "" { + writeError(w, fmt.Errorf("Invalid Host")) + return + } + + service := vars["serv_name"] + + useTLS, _ := strconv.ParseBool(r.Header.Get("use_tls")) + restart, _ := strconv.ParseBool(r.FormValue("restart")) + + // limit upload file to 5mb + err := r.ParseMultipartForm(5 << 20) + if err != nil { + writeError(w, err) + return + } + + // convert uploaded files to list of Proto struct + files := r.MultipartForm.File["protos"] + protos := make([]core.Proto, 0, len(files)) + for _, file := range files { + fileData, err := file.Open() + if err != nil { + writeError(w, err) + return + } + defer fileData.Close() + + content, err := ioutil.ReadAll(fileData) + if err != nil { + writeError(w, err) + } + + protos = append(protos, core.Proto{ + Name: file.Filename, + Content: content, + }) + } + + res, err := h.g.GetResourceWithProto(context.Background(), host, !useTLS, restart, protos) + if err != nil { + writeError(w, err) + return + } + + result, err := res.List(service) + if err != nil { + writeError(w, err) + return + } + + h.g.Extend(host) + response(w, result) +} + func (h *Handler) describeFunction(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) host := vars["host"] diff --git a/handler/routes.go b/handler/routes.go index e05a89f..4e5c27c 100644 --- a/handler/routes.go +++ b/handler/routes.go @@ -14,6 +14,7 @@ func Init(router *mux.Router) { ajaxRoute := router.PathPrefix("/server/{host}").Subrouter() ajaxRoute.HandleFunc("/services", corsHandler(h.getLists)).Methods(http.MethodGet, http.MethodOptions) + ajaxRoute.HandleFunc("/services", corsHandler(h.getListsWithProto)).Methods(http.MethodPost) ajaxRoute.HandleFunc("/service/{serv_name}/functions", corsHandler(h.getLists)).Methods(http.MethodGet, http.MethodOptions) ajaxRoute.HandleFunc("/function/{func_name}/describe", corsHandler(h.describeFunction)).Methods(http.MethodGet, http.MethodOptions) ajaxRoute.HandleFunc("/function/{func_name}/invoke", corsHandler(h.invokeFunction)).Methods(http.MethodPost, http.MethodOptions) diff --git a/index/css/proto.css b/index/css/proto.css new file mode 100644 index 0000000..6e2d15c --- /dev/null +++ b/index/css/proto.css @@ -0,0 +1,115 @@ +#proto-input { + margin-top: 10px; + margin-bottom: 15px; +} + +[type="file"] { + border: 0; + clip: rect(0, 0, 0, 0); + height: 1px; + overflow: hidden; + padding: 0; + position: absolute !important; + white-space: nowrap; + width: 1px; +} +[type="file"] + label { + background-color: #59698d; + border-radius: 10px; + color: #fff; + cursor: pointer; + display: inline-block; + font-family: 'Poppins', sans-serif; + font-size: 1rem; + font-weight: 600; + padding: 5px; + transition: background-color 0.3s; +} + +[type="file"]:focus + label, +[type="file"] + label:hover { + background-color: #324674; +} + +[type="file"]:focus + label { + outline: 1px dotted #000; + outline: -webkit-focus-ring-color auto 5px; +} + +.proto-top-collection { + display: inline-flex; + flex-direction: row; + align-items: center; + justify-content: space-between; + width: 100%; +} + +.proto-toggle { + color: rgba(0, 0, 0, 0.5); + font-size: 0.8rem; +} + +.proto-toggle:hover { + cursor: pointer; +} + +.proto-collection { + background-color: #e0dfe6; + border-radius: 5px; + min-height: 120px; + width: 100%; + display: inline-flex; + align-items: center; + align-content: center; + flex-direction: row; + padding: 20px; + padding-right: 0px; + flex-wrap: wrap; +} + +.proto-item { + height: 80px; + margin-right: 20px; + margin-left: 10px; + margin-bottom: 30px; + + display: inline-flex; + flex-direction: row; + align-items: flex-start; +} + +.proto-icon { + height: 80px; + opacity: 0.5; +} + +.proto-desc { + display: inline-flex; + flex-direction: column; + align-items: flex-start; + height: 100%; +} + +.proto-caption { + font-weight: 600; + color: rgba(0,0,0,0.6); + font-size: 1rem; +} + +.proto-size { + font-size: 1rem; + color: rgba(0,0,0,0.6); +} + +.proto-remove { + background-color: rgba(0,0,0,0.5); + padding: 3px !important; + margin-left: 0px !important; + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; +} + +.proto-remove:hover { + background-color: rgba(255,0,0,0.4); +} diff --git a/index/img/file.png b/index/img/file.png new file mode 100644 index 0000000..131f2f6 Binary files /dev/null and b/index/img/file.png differ diff --git a/index/index.html b/index/index.html index 81d4ca4..cf77705 100644 --- a/index/index.html +++ b/index/index.html @@ -10,6 +10,7 @@ + @@ -28,6 +29,24 @@ +
+
+ + +
+
+ + +