实现http请求响应转发

cmd/build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -x
+set -e
+
+CURRENT=`pwd`
+
+#CGO_ENABLED=0 GOOS=linux GOARCH=amd64
+
+cd $(dirname $0)
+for name in `ls -d */|sed 's/\///g'`
+do
+  CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "../bin/$name" ./"$name"
+done
+
+ls ../bin/
+cd "$CURRENT"

go.mod

@@ -1,6 +1,6 @@
 module github.com/ginuerzh/gost
 
-go 1.22
+go 1.21
 
 replace github.com/templexxx/cpu v0.0.7 => github.com/templexxx/cpu v0.0.10-0.20211111114238-98168dcec14a
 
@@ -14,6 +14,7 @@ require (
 	github.com/go-gost/tls-dissector v0.0.2-0.20220408131628-aac992c27451
 	github.com/go-log/log v0.2.0
 	github.com/gobwas/glob v0.2.3
+	github.com/google/uuid v1.6.0
 	github.com/gorilla/websocket v1.5.1
 	github.com/klauspost/compress v1.17.6
 	github.com/mdlayher/vsock v1.2.1
@@ -29,6 +30,7 @@ require (
 	gitlab.com/yawning/obfs4.git v0.0.0-20220204003609-77af0cba934d
 	golang.org/x/crypto v0.24.0
 	golang.org/x/net v0.26.0
+	golang.org/x/text v0.16.0
 )
 
 require (
@@ -55,6 +57,5 @@ require (
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/tools v0.22.0 // indirect
 )

go.sum

@@ -61,6 +61,8 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g=
 github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
 github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
 github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=

gost.go

@@ -27,9 +27,10 @@ var Debug bool
 
 var (
 	tinyBufferSize   = 512
-	smallBufferSize  = 2 * 1024  // 2KB small buffer
+	smallBufferSize  = 2 * 1024         // 2KB small buffer
-	mediumBufferSize = 8 * 1024  // 8KB medium buffer
+	mediumBufferSize = 8 * 1024         // 8KB medium buffer
-	largeBufferSize  = 32 * 1024 // 32KB large buffer
+	largeBufferSize  = 32 * 1024        // 32KB large buffer
+	bigBufferSize    = 10 * 1024 * 1024 // 32KB large buffer
 )
 
 var (
@@ -48,6 +49,12 @@ var (
 			return make([]byte, largeBufferSize)
 		},
 	}
+
+	bigRespPool = sync.Pool{
+		New: func() interface{} {
+			return make([]byte, bigBufferSize)
+		},
+	}
 )
 
 var (

server.go

@@ -1,13 +1,29 @@
 package gost
 
 import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"golang.org/x/net/html/charset"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/unicode"
+	"golang.org/x/text/transform"
 	"io"
+	"io/ioutil"
 	"net"
+	"net/http"
+	"os"
+	"strings"
 	"time"
 
 	"github.com/go-log/log"
+
+	"github.com/google/uuid"
 )
 
+var dataFile *os.File
+
 // Accepter represents a network endpoint that can accept connection from peer.
 type Accepter interface {
 	Accept() (net.Conn, error)
@@ -28,6 +44,13 @@ func (s *Server) Init(opts ...ServerOption) {
 	for _, opt := range opts {
 		opt(s.options)
 	}
+
+	f, err := os.Create("./data.txt")
+	if err != nil {
+		fmt.Println("无法创建文件:", err)
+		return
+	}
+	dataFile = f
 }
 
 // Addr returns the address of the server
@@ -102,20 +125,189 @@ type Listener interface {
 	net.Listener
 }
 
+type HttpMessage struct {
+	RequestID   string
+	RawRequest  []byte
+	RawResponse []byte
+	Request     *http.Request
+	Response    *http.Response
+}
+
 func transport(rw1, rw2 io.ReadWriter) error {
+	requestID := uuid.NewString()
+	var httpMsg = HttpMessage{
+		RequestID: requestID,
+	}
+	log.Logf("transport requestID:%s\n", requestID)
 	errc := make(chan error, 1)
 	go func() {
-		errc <- copyBuffer(rw1, rw2)
+		errc <- copyResponseBuffer(rw1, rw2, &httpMsg)
 	}()
 
 	go func() {
-		errc <- copyBuffer(rw2, rw1)
+		errc <- copyRequestBuffer(rw2, rw1, &httpMsg)
 	}()
 
 	if err := <-errc; err != nil && err != io.EOF {
 		return err
 	}
 
+	//var err error
+	//var wg sync.WaitGroup
+	//wg.Add(1)
+	//go func() {
+	//	defer wg.Done()
+	//	err = copyResponseBuffer(rw1, rw2, &httpMsg)
+	//}()
+	//
+	//wg.Add(1)
+	//go func() {
+	//	defer wg.Done()
+	//	err = copyRequestBuffer(rw2, rw1, &httpMsg)
+	//}()
+	//
+	//wg.Wait()
+	//if err != nil && err != io.EOF {
+	//	return err
+	//}
+
+	//if len(httpMsg.ReqBody) > 0 {
+	//	log.Logf("recieve http msg reqeustID:%s \n", requestID)
+	//	reqMsgContent, err := parseMessageContent(httpMsg.ReqBody)
+	//	if err != nil {
+	//		log.Logf("parse http request message failed:%+v", err)
+	//	}
+	//	log.Logf("http request body reqeustID:%s\n%s", requestID, reqMsgContent)
+	//	respMsgContent, err := parseMessageContent(httpMsg.RespBody)
+	//	if err != nil {
+	//		log.Logf("parse http response message failed:%+v", err)
+	//	}
+	//	log.Logf("http response body reqeustID:%s \n%s", requestID, respMsgContent)
+	//}
+
+	return nil
+}
+
+func copyRequestBuffer(dst io.Writer, src io.Reader, httpMsg *HttpMessage) error {
+	buf := lPool.Get().([]byte)
+	defer lPool.Put(buf)
+
+	_, err, all_buf := CustomCopyBuffer(dst, src, buf, "request")
+	if err != nil {
+		log.Logf("copyRequestBuffer failed:%+v", err)
+		errMsg := err.Error()
+		validErrMsgArr := []string{"EOF", "use of closed network connection"}
+		var validErrFlag bool = false
+		for _, validErrMsg := range validErrMsgArr {
+			if strings.Contains(errMsg, validErrMsg) {
+				validErrFlag = true
+				break
+			}
+		}
+
+		if !validErrFlag {
+			return err
+		}
+	}
+
+	buf_content := string(all_buf)
+	if isHTTPMessage(buf_content) {
+		httpMsg.RawRequest = all_buf
+	}
+	//log.Logf("copyRequestBuffer data:\n%s", buf_content)
+	// dataFile.WriteString("request:" + buf_content + "\n")
+
+	return nil
+}
+
+func isHTTPMessage(socks5Message string) bool {
+	// 检查 SOCKS5 报文是否以 HTTP 请求方法开头
+	httpMethods := []string{"GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"} // 常见的 HTTP 请求方法
+	for _, method := range httpMethods {
+		if strings.HasPrefix(socks5Message, method) {
+			return true
+		}
+	}
+	return false
+}
+
+func parseMessageContent(buf []byte) (string, error) {
+	//获取响应体
+	bodyReader := bufio.NewReader(bytes.NewReader(buf))
+	//使用determiEncoding函数对获取的信息进行解析
+	e := determineEncoding(bodyReader)
+	utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
+
+	//读取并打印获取的信息
+	resultBytes, err := ioutil.ReadAll(utf8Reader)
+	if err != nil {
+		log.Logf("utf8Reader failed:%+v", err)
+		return "", err
+	}
+	content := string(resultBytes)
+	return content, nil
+}
+
+func copyResponseBuffer(dst io.Writer, src io.Reader, httpMsg *HttpMessage) error {
+	buf := bigRespPool.Get().([]byte)
+	defer bigRespPool.Put(buf)
+
+	t1 := time.Now().UnixNano() / 1e6
+	_, err, all_buf := CustomCopyBuffer(dst, src, buf, "response")
+	t2 := time.Now().UnixNano() / 1e6
+	if err != nil {
+		log.Logf("copyResponseBuffer failed:%+v", err)
+		errMsg := err.Error()
+		validErrMsgArr := []string{"EOF", "use of closed network connection"}
+		var validErrFlag bool = false
+		for _, validErrMsg := range validErrMsgArr {
+			if strings.Contains(errMsg, validErrMsg) {
+				validErrFlag = true
+				break
+			}
+		}
+
+		if !validErrFlag {
+			return err
+		}
+	}
+
+	log.Logf("copyResponseBuffer CustomCopyBuffer during:%dms", (t2 - t1))
+	if httpMsg.RawRequest != nil && len(httpMsg.RawRequest) > 0 {
+		httpMsg.RawResponse = all_buf
+		requestID := httpMsg.RequestID
+		log.Logf("recieve http msg reqeustID:%s \n", requestID)
+		reqMsgContent, err := parseMessageContent(httpMsg.RawRequest)
+		if err != nil {
+			log.Logf("parse http request message failed:%+v", err)
+		}
+		log.Logf("http request body reqeustID:%s\n%s", requestID, reqMsgContent)
+		respMsgContent, err := parseMessageContent(httpMsg.RawResponse)
+		if err != nil {
+			log.Logf("parse http response message failed:%+v", err)
+		}
+		log.Logf("http response body reqeustID:%s \n%s", requestID, respMsgContent)
+
+		// 解析http request 对象
+		requestReader := bufio.NewReader(bytes.NewReader(httpMsg.RawRequest))
+		request, err := http.ReadRequest(requestReader)
+		if err != nil {
+			log.Logf("parse HTTP request faild:%+v", err)
+			return nil
+		}
+
+		// 解析http response对象
+		responseReader := bufio.NewReader(bytes.NewReader(httpMsg.RawResponse))
+		response, err := http.ReadResponse(responseReader, request)
+		if err != nil {
+			log.Logf("parse HTTP response faild:%+v", err)
+			return nil
+		}
+		httpMsg.Request = request
+		httpMsg.Response = response
+		log.Logf("----------parse http request and response successfully---------")
+	}
+
 	return nil
 }
 
@@ -124,5 +316,95 @@ func copyBuffer(dst io.Writer, src io.Reader) error {
 	defer lPool.Put(buf)
 
 	_, err := io.CopyBuffer(dst, src, buf)
+	if err != nil {
+		return nil
+	}
+
+	buf_trim := bytes.Trim(buf, "\x00")
+	buf_content := string(buf_trim)
+	log.Logf("copyBuffer data:\n%s", buf_content)
+
 	return err
 }
+
+// 解析编码格式
+func determineEncoding(r *bufio.Reader) encoding.Encoding {
+	bytes, err := r.Peek(1024)
+	if err != nil {
+		log.Logf("Fetcher error: %v", err)
+		return unicode.UTF8
+	}
+	e, _, _ := charset.DetermineEncoding(
+		bytes, "")
+	return e
+}
+
+func CustomCopyBuffer(dst io.Writer, src io.Reader, buf []byte, reqType string) (written int64, err error, all_buf []byte) {
+	if buf != nil && len(buf) == 0 {
+		panic("empty buffer in CopyBuffer")
+	}
+	return customCopyBuffer(dst, src, buf, reqType)
+}
+
+// copyBuffer is the actual implementation of Copy and CopyBuffer.
+// if buf is nil, one is allocated.
+func customCopyBuffer(dst io.Writer, src io.Reader, buf []byte, reqType string) (written int64, err error, all_buf []byte) {
+	// If the reader has a WriteTo method, use it to do the copy.
+	// Avoids an allocation and a copy.
+	all_buf = make([]byte, 0)
+	if wt, ok := src.(io.WriterTo); ok {
+		n, err := wt.WriteTo(dst)
+		return n, err, nil
+	}
+
+	// Similarly, if the writer has a ReadFrom method, use it to do the copy.
+	if rt, ok := dst.(io.ReaderFrom); ok {
+		n, err := rt.ReadFrom(src)
+		return n, err, nil
+	}
+
+	if buf == nil {
+		size := 32 * 1024
+		if l, ok := src.(*io.LimitedReader); ok && int64(size) > l.N {
+			if l.N < 1 {
+				size = 1
+			} else {
+				size = int(l.N)
+			}
+		}
+		buf = make([]byte, size)
+	}
+
+	for {
+		nr, er := src.Read(buf)
+		if nr > 0 {
+			dataBytes := buf[0:nr]
+			nw, ew := dst.Write(dataBytes)
+			all_buf = append(all_buf, dataBytes...)
+			if nw < 0 || nr < nw {
+				nw = 0
+				if ew == nil {
+					ew = errors.New("invalid write result")
+				}
+			}
+			written += int64(nw)
+			if ew != nil {
+				err = ew
+				break
+			}
+			if nr != nw {
+				err = errors.New("short write")
+				break
+			}
+		}
+
+		if er != nil {
+			if er != errors.New("EOF") {
+				err = er
+			}
+			break
+		}
+	}
+
+	return written, err, all_buf
+}