code refactoring

addon-dailer
lqqyt2423 2 years ago
parent 7059c4e8b3
commit 01f0cc7a4b

@ -1,88 +0,0 @@
package addon
import (
"time"
"github.com/lqqyt2423/go-mitmproxy/connection"
"github.com/lqqyt2423/go-mitmproxy/flow"
_log "github.com/sirupsen/logrus"
)
var log = _log.WithField("at", "addon")
type Addon interface {
// A client has connected to mitmproxy. Note that a connection can correspond to multiple HTTP requests.
ClientConnected(*connection.Client)
// A client connection has been closed (either by us or the client).
ClientDisconnected(*connection.Client)
// Mitmproxy has connected to a server.
ServerConnected(*flow.ConnContext)
// A server connection has been closed (either by us or the server).
ServerDisconnected(*flow.ConnContext)
// HTTP request headers were successfully read. At this point, the body is empty.
Requestheaders(*flow.Flow)
// The full HTTP request has been read.
Request(*flow.Flow)
// HTTP response headers were successfully read. At this point, the body is empty.
Responseheaders(*flow.Flow)
// The full HTTP response has been read.
Response(*flow.Flow)
}
// Base do nothing
type Base struct{}
func (addon *Base) ClientConnected(*connection.Client) {}
func (addon *Base) ClientDisconnected(*connection.Client) {}
func (addon *Base) ServerConnected(*flow.ConnContext) {}
func (addon *Base) ServerDisconnected(*flow.ConnContext) {}
func (addon *Base) Requestheaders(*flow.Flow) {}
func (addon *Base) Request(*flow.Flow) {}
func (addon *Base) Responseheaders(*flow.Flow) {}
func (addon *Base) Response(*flow.Flow) {}
// Log log http record
type Log struct {
Base
}
func (addon *Log) ClientConnected(client *connection.Client) {
log.Infof("%v client connect\n", client.Conn.RemoteAddr())
}
func (addon *Log) ClientDisconnected(client *connection.Client) {
log.Infof("%v client disconnect\n", client.Conn.RemoteAddr())
}
func (addon *Log) ServerConnected(connCtx *flow.ConnContext) {
log.Infof("%v server connect %v (%v)\n", connCtx.Client.Conn.RemoteAddr(), connCtx.Server.Address, connCtx.Server.Conn.RemoteAddr())
}
func (addon *Log) ServerDisconnected(connCtx *flow.ConnContext) {
log.Infof("%v server disconnect %v (%v)\n", connCtx.Client.Conn.RemoteAddr(), connCtx.Server.Address, connCtx.Server.Conn.RemoteAddr())
}
func (addon *Log) Requestheaders(f *flow.Flow) {
log := log.WithField("in", "Log")
start := time.Now()
go func() {
<-f.Done()
var StatusCode int
if f.Response != nil {
StatusCode = f.Response.StatusCode
}
var contentLen int
if f.Response != nil && f.Response.Body != nil {
contentLen = len(f.Response.Body)
}
log.Infof("%v %v %v %v %v - %v ms\n", f.ConnContext.Client.Conn.RemoteAddr(), f.Request.Method, f.Request.URL.String(), StatusCode, contentLen, time.Since(start).Milliseconds())
}()
}

@ -1,13 +1,13 @@
package addon package addon
import "github.com/lqqyt2423/go-mitmproxy/flow" import "github.com/lqqyt2423/go-mitmproxy/proxy"
// decode content-encoding then respond to client // decode content-encoding then respond to client
type Decoder struct { type Decoder struct {
Base proxy.BaseAddon
} }
func (d *Decoder) Response(f *flow.Flow) { func (d *Decoder) Response(f *proxy.Flow) {
f.Response.ReplaceToDecodedBody() f.Response.ReplaceToDecodedBody()
} }

@ -9,30 +9,40 @@ import (
"strings" "strings"
"unicode" "unicode"
"github.com/lqqyt2423/go-mitmproxy/flow" "github.com/lqqyt2423/go-mitmproxy/proxy"
log "github.com/sirupsen/logrus"
) )
type Dumper struct { type Dumper struct {
Base proxy.BaseAddon
out io.Writer
level int // 0: header 1: header + body level int // 0: header 1: header + body
Out io.Writer
} }
func NewDumper(file string, level int) *Dumper { func NewDumper(out io.Writer, level int) *Dumper {
out, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
panic(err)
}
if level != 0 && level != 1 { if level != 0 && level != 1 {
level = 0 level = 0
} }
return &Dumper{out: out, level: level}
}
func NewDumperWithFilename(filename string, level int) *Dumper {
out, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
panic(err)
}
return NewDumper(out, level)
}
return &Dumper{Out: out, level: level} func (d *Dumper) Requestheaders(f *proxy.Flow) {
go func() {
<-f.Done()
d.dump(f)
}()
} }
// call when <-f.Done() // call when <-f.Done()
func (d *Dumper) dump(f *flow.Flow) { func (d *Dumper) dump(f *proxy.Flow) {
// 参考 httputil.DumpRequest // 参考 httputil.DumpRequest
log := log.WithField("in", "Dumper") log := log.WithField("in", "Dumper")
@ -53,7 +63,7 @@ func (d *Dumper) dump(f *flow.Flow) {
} }
buf.WriteString("\r\n") buf.WriteString("\r\n")
if d.level == 1 && f.Request.Body != nil && len(f.Request.Body) > 0 && CanPrint(f.Request.Body) { if d.level == 1 && f.Request.Body != nil && len(f.Request.Body) > 0 && canPrint(f.Request.Body) {
buf.Write(f.Request.Body) buf.Write(f.Request.Body)
buf.WriteString("\r\n\r\n") buf.WriteString("\r\n\r\n")
} }
@ -77,20 +87,13 @@ func (d *Dumper) dump(f *flow.Flow) {
buf.WriteString("\r\n\r\n") buf.WriteString("\r\n\r\n")
_, err = d.Out.Write(buf.Bytes()) _, err = d.out.Write(buf.Bytes())
if err != nil { if err != nil {
log.Error(err) log.Error(err)
} }
} }
func (d *Dumper) Requestheaders(f *flow.Flow) { func canPrint(content []byte) bool {
go func() {
<-f.Done()
d.dump(f)
}()
}
func CanPrint(content []byte) bool {
for _, c := range string(content) { for _, c := range string(content) {
if !unicode.IsPrint(c) && !unicode.IsSpace(c) { if !unicode.IsPrint(c) && !unicode.IsSpace(c) {
return false return false

@ -1,96 +0,0 @@
package flowmapper
import (
"io/ioutil"
"path/filepath"
"regexp"
"strings"
"github.com/lqqyt2423/go-mitmproxy/addon"
"github.com/lqqyt2423/go-mitmproxy/flow"
_log "github.com/sirupsen/logrus"
)
var log = _log.WithField("at", "changeflow addon")
var httpsRegexp = regexp.MustCompile(`^https://`)
type Mapper struct {
addon.Base
reqResMap map[string]*flow.Response
}
func NewMapper(dirname string) *Mapper {
infos, err := ioutil.ReadDir(dirname)
if err != nil {
panic(err)
}
filenames := make([]string, 0)
for _, info := range infos {
if info.IsDir() {
continue
}
if !strings.HasSuffix(info.Name(), ".map.txt") {
continue
}
filenames = append(filenames, filepath.Join(dirname, info.Name()))
}
if len(filenames) == 0 {
return &Mapper{
reqResMap: make(map[string]*flow.Response),
}
}
ch := make(chan interface{}, len(filenames))
for _, filename := range filenames {
go func(filename string, ch chan<- interface{}) {
f, err := ParseFlowFromFile(filename)
if err != nil {
log.Error(err)
ch <- err
return
}
ch <- f
}(filename, ch)
}
reqResMap := make(map[string]*flow.Response)
for i := 0; i < len(filenames); i++ {
flowOrErr := <-ch
if f, ok := flowOrErr.(*flow.Flow); ok {
key := buildReqKey(f.Request)
log.Infof("add request mapper: %v", key)
reqResMap[key] = f.Response
}
}
return &Mapper{
reqResMap: reqResMap,
}
}
func ParseFlowFromFile(filename string) (*flow.Flow, error) {
p, err := NewParserFromFile(filename)
if err != nil {
return nil, err
}
return p.Parse()
}
func (c *Mapper) Request(f *flow.Flow) {
key := buildReqKey(f.Request)
if resp, ok := c.reqResMap[key]; ok {
f.Response = resp
}
}
func buildReqKey(req *flow.Request) string {
url := req.URL.String()
url = httpsRegexp.ReplaceAllString(url, "http://")
key := req.Method + " " + url
return key
}

@ -1,46 +1,131 @@
package flowmapper package addon
import ( import (
"errors" "errors"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"path/filepath"
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"github.com/lqqyt2423/go-mitmproxy/flow" "github.com/lqqyt2423/go-mitmproxy/proxy"
log "github.com/sirupsen/logrus"
) )
type Parser struct { var httpsRegexp = regexp.MustCompile(`^https://`)
type Mapper struct {
proxy.BaseAddon
reqResMap map[string]*proxy.Response
}
func NewMapper(dirname string) *Mapper {
infos, err := ioutil.ReadDir(dirname)
if err != nil {
panic(err)
}
filenames := make([]string, 0)
for _, info := range infos {
if info.IsDir() {
continue
}
if !strings.HasSuffix(info.Name(), ".map.txt") {
continue
}
filenames = append(filenames, filepath.Join(dirname, info.Name()))
}
if len(filenames) == 0 {
return &Mapper{
reqResMap: make(map[string]*proxy.Response),
}
}
ch := make(chan interface{}, len(filenames))
for _, filename := range filenames {
go func(filename string, ch chan<- interface{}) {
f, err := parseFlowFromFile(filename)
if err != nil {
log.Error(err)
ch <- err
return
}
ch <- f
}(filename, ch)
}
reqResMap := make(map[string]*proxy.Response)
for i := 0; i < len(filenames); i++ {
flowOrErr := <-ch
if f, ok := flowOrErr.(*proxy.Flow); ok {
key := buildReqKey(f.Request)
log.Infof("add request mapper: %v", key)
reqResMap[key] = f.Response
}
}
return &Mapper{
reqResMap: reqResMap,
}
}
func parseFlowFromFile(filename string) (*proxy.Flow, error) {
p, err := newMapperParserFromFile(filename)
if err != nil {
return nil, err
}
return p.parse()
}
func (c *Mapper) Request(f *proxy.Flow) {
key := buildReqKey(f.Request)
if resp, ok := c.reqResMap[key]; ok {
f.Response = resp
}
}
func buildReqKey(req *proxy.Request) string {
url := req.URL.String()
url = httpsRegexp.ReplaceAllString(url, "http://")
key := req.Method + " " + url
return key
}
type mapperParser struct {
lines []string lines []string
url string url string
request *flow.Request request *proxy.Request
response *flow.Response response *proxy.Response
} }
func NewParserFromFile(filename string) (*Parser, error) { func newMapperParserFromFile(filename string) (*mapperParser, error) {
bytes, err := ioutil.ReadFile(filename) bytes, err := ioutil.ReadFile(filename)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return NewParserFromString(string(bytes)) return newMapperParserFromString(string(bytes))
} }
func NewParserFromString(content string) (*Parser, error) { func newMapperParserFromString(content string) (*mapperParser, error) {
content = strings.TrimSpace(content) content = strings.TrimSpace(content)
lines := strings.Split(content, "\n") lines := strings.Split(content, "\n")
if len(lines) == 0 { if len(lines) == 0 {
return nil, errors.New("no lines") return nil, errors.New("no lines")
} }
return &Parser{ return &mapperParser{
lines: lines, lines: lines,
}, nil }, nil
} }
func (p *Parser) Parse() (*flow.Flow, error) { func (p *mapperParser) parse() (*proxy.Flow, error) {
if err := p.parseRequest(); err != nil { if err := p.parseRequest(); err != nil {
return nil, err return nil, err
} }
@ -49,13 +134,13 @@ func (p *Parser) Parse() (*flow.Flow, error) {
return nil, err return nil, err
} }
return &flow.Flow{ return &proxy.Flow{
Request: p.request, Request: p.request,
Response: p.response, Response: p.response,
}, nil }, nil
} }
func (p *Parser) parseRequest() error { func (p *mapperParser) parseRequest() error {
if err := p.parseReqHead(); err != nil { if err := p.parseReqHead(); err != nil {
return err return err
} }
@ -85,7 +170,7 @@ func (p *Parser) parseRequest() error {
return nil return nil
} }
func (p *Parser) parseReqHead() error { func (p *mapperParser) parseReqHead() error {
line, _ := p.getLine() line, _ := p.getLine()
re := regexp.MustCompile(`^(GET|POST|PUT|DELETE)\s+?(.+)`) re := regexp.MustCompile(`^(GET|POST|PUT|DELETE)\s+?(.+)`)
matches := re.FindStringSubmatch(line) matches := re.FindStringSubmatch(line)
@ -93,7 +178,7 @@ func (p *Parser) parseReqHead() error {
return errors.New("request head parse error") return errors.New("request head parse error")
} }
p.request = &flow.Request{ p.request = &proxy.Request{
Method: matches[1], Method: matches[1],
} }
p.url = matches[2] p.url = matches[2]
@ -101,7 +186,7 @@ func (p *Parser) parseReqHead() error {
return nil return nil
} }
func (p *Parser) parseHeader() (http.Header, error) { func (p *mapperParser) parseHeader() (http.Header, error) {
header := make(http.Header) header := make(http.Header)
re := regexp.MustCompile(`^([\w-]+?):\s*(.+)$`) re := regexp.MustCompile(`^([\w-]+?):\s*(.+)$`)
@ -127,7 +212,7 @@ func (p *Parser) parseHeader() (http.Header, error) {
return header, nil return header, nil
} }
func (p *Parser) parseReqBody() { func (p *mapperParser) parseReqBody() {
bodyLines := make([]string, 0) bodyLines := make([]string, 0)
for { for {
@ -155,7 +240,7 @@ func (p *Parser) parseReqBody() {
p.request.Body = []byte(body) p.request.Body = []byte(body)
} }
func (p *Parser) parseResponse() error { func (p *mapperParser) parseResponse() error {
if err := p.parseResHead(); err != nil { if err := p.parseResHead(); err != nil {
return err return err
} }
@ -175,7 +260,7 @@ func (p *Parser) parseResponse() error {
return nil return nil
} }
func (p *Parser) parseResHead() error { func (p *mapperParser) parseResHead() error {
line, ok := p.getLine() line, ok := p.getLine()
if !ok { if !ok {
return errors.New("response no head line") return errors.New("response no head line")
@ -188,14 +273,14 @@ func (p *Parser) parseResHead() error {
} }
code, _ := strconv.Atoi(matches[1]) code, _ := strconv.Atoi(matches[1])
p.response = &flow.Response{ p.response = &proxy.Response{
StatusCode: code, StatusCode: code,
} }
return nil return nil
} }
func (p *Parser) getLine() (string, bool) { func (p *mapperParser) getLine() (string, bool) {
if len(p.lines) == 0 { if len(p.lines) == 0 {
return "", false return "", false
} }

@ -1,4 +1,4 @@
package flowmapper package addon
import "testing" import "testing"
@ -14,11 +14,11 @@ HTTP/1.1 200
ok ok
` `
p, err := NewParserFromString(content) p, err := newMapperParserFromString(content)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
f, err := p.Parse() f, err := p.parse()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

@ -7,9 +7,8 @@ import (
"os" "os"
"github.com/lqqyt2423/go-mitmproxy/addon" "github.com/lqqyt2423/go-mitmproxy/addon"
"github.com/lqqyt2423/go-mitmproxy/addon/flowmapper"
"github.com/lqqyt2423/go-mitmproxy/addon/web"
"github.com/lqqyt2423/go-mitmproxy/proxy" "github.com/lqqyt2423/go-mitmproxy/proxy"
"github.com/lqqyt2423/go-mitmproxy/web"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
@ -82,16 +81,16 @@ func main() {
log.Infof("go-mitmproxy version %v\n", p.Version) log.Infof("go-mitmproxy version %v\n", p.Version)
p.AddAddon(&addon.Log{}) p.AddAddon(&proxy.LogAddon{})
p.AddAddon(web.NewWebAddon(config.webAddr)) p.AddAddon(web.NewWebAddon(config.webAddr))
if config.dump != "" { if config.dump != "" {
dumper := addon.NewDumper(config.dump, config.dumpLevel) dumper := addon.NewDumperWithFilename(config.dump, config.dumpLevel)
p.AddAddon(dumper) p.AddAddon(dumper)
} }
if config.mapperDir != "" { if config.mapperDir != "" {
mapper := flowmapper.NewMapper(config.mapperDir) mapper := addon.NewMapper(config.mapperDir)
p.AddAddon(mapper) p.AddAddon(mapper)
} }

@ -1,35 +0,0 @@
package connection
import (
"net"
"net/http"
uuid "github.com/satori/go.uuid"
)
type Client struct {
Id uuid.UUID
Conn net.Conn
Tls bool
}
func NewClient(c net.Conn) *Client {
return &Client{
Id: uuid.NewV4(),
Conn: c,
Tls: false,
}
}
type Server struct {
Id uuid.UUID
Conn net.Conn
Client *http.Client
Address string
}
func NewServer() *Server {
return &Server{
Id: uuid.NewV4(),
}
}

@ -5,20 +5,17 @@ import (
"strconv" "strconv"
"strings" "strings"
log "github.com/sirupsen/logrus"
"github.com/lqqyt2423/go-mitmproxy/addon"
"github.com/lqqyt2423/go-mitmproxy/flow"
"github.com/lqqyt2423/go-mitmproxy/proxy" "github.com/lqqyt2423/go-mitmproxy/proxy"
log "github.com/sirupsen/logrus"
) )
var titleRegexp = regexp.MustCompile("(<title>)(.*?)(</title>)") var titleRegexp = regexp.MustCompile("(<title>)(.*?)(</title>)")
type ChangeHtml struct { type ChangeHtml struct {
addon.Base proxy.BaseAddon
} }
func (c *ChangeHtml) Response(f *flow.Flow) { func (c *ChangeHtml) Response(f *proxy.Flow) {
contentType := f.Response.Header.Get("Content-Type") contentType := f.Response.Header.Get("Content-Type")
if !strings.Contains(contentType, "text/html") { if !strings.Contains(contentType, "text/html") {
return return

@ -5,17 +5,15 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/lqqyt2423/go-mitmproxy/addon"
"github.com/lqqyt2423/go-mitmproxy/flow"
"github.com/lqqyt2423/go-mitmproxy/proxy" "github.com/lqqyt2423/go-mitmproxy/proxy"
) )
type AddHeader struct { type AddHeader struct {
addon.Base proxy.BaseAddon
count int count int
} }
func (a *AddHeader) Responseheaders(f *flow.Flow) { func (a *AddHeader) Responseheaders(f *proxy.Flow) {
a.count += 1 a.count += 1
f.Response.Header.Add("x-count", strconv.Itoa(a.count)) f.Response.Header.Add("x-count", strconv.Itoa(a.count))
} }

@ -1,119 +0,0 @@
package flow
import (
"context"
"crypto/tls"
"net"
"net/http"
"github.com/lqqyt2423/go-mitmproxy/connection"
)
var ConnContextKey = new(struct{})
type ConnContext struct {
Client *connection.Client
Server *connection.Server
}
func NewConnContext(c net.Conn) *ConnContext {
client := connection.NewClient(c)
return &ConnContext{
Client: client,
}
}
func (connCtx *ConnContext) InitHttpServer(sslInsecure bool, connWrap func(net.Conn) net.Conn, whenConnected func()) {
if connCtx.Server != nil {
return
}
if connCtx.Client.Tls {
return
}
server := connection.NewServer()
server.Client = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
// todo: change here
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
c, err := (&net.Dialer{
// Timeout: 30 * time.Second,
// KeepAlive: 30 * time.Second,
}).DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
cw := connWrap(c)
server.Conn = cw
server.Address = addr
defer whenConnected()
return cw, nil
},
ForceAttemptHTTP2: false, // disable http2
DisableCompression: true, // To get the original response from the server, set Transport.DisableCompression to true.
TLSClientConfig: &tls.Config{
InsecureSkipVerify: sslInsecure,
KeyLogWriter: GetTlsKeyLogWriter(),
},
},
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// 禁止自动重定向
return http.ErrUseLastResponse
},
}
connCtx.Server = server
}
func (connCtx *ConnContext) InitHttpsServer(sslInsecure bool, connWrap func(net.Conn) net.Conn, whenConnected func()) {
if connCtx.Server != nil {
return
}
if !connCtx.Client.Tls {
return
}
server := connection.NewServer()
server.Client = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
log.Debugln("in https DialTLSContext")
plainConn, err := (&net.Dialer{}).DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
cw := connWrap(plainConn)
server.Conn = cw
server.Address = addr
whenConnected()
firstTLSHost, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
cfg := &tls.Config{
InsecureSkipVerify: sslInsecure,
KeyLogWriter: GetTlsKeyLogWriter(),
ServerName: firstTLSHost,
}
tlsConn := tls.Client(cw, cfg)
return tlsConn, nil
},
ForceAttemptHTTP2: false, // disable http2
DisableCompression: true, // To get the original response from the server, set Transport.DisableCompression to true.
},
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// 禁止自动重定向
return http.ErrUseLastResponse
},
}
connCtx.Server = server
}

@ -1,29 +0,0 @@
package flow
import (
"io"
"os"
"sync"
)
// Wireshark 解析 https 设置
var tlsKeyLogWriter io.Writer
var tlsKeyLogOnce sync.Once
func GetTlsKeyLogWriter() io.Writer {
tlsKeyLogOnce.Do(func() {
logfile := os.Getenv("SSLKEYLOGFILE")
if logfile == "" {
return
}
writer, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.WithField("in", "GetTlsKeyLogWriter").Debug(err)
return
}
tlsKeyLogWriter = writer
})
return tlsKeyLogWriter
}

@ -0,0 +1,81 @@
package proxy
import (
"time"
)
type Addon interface {
// A client has connected to mitmproxy. Note that a connection can correspond to multiple HTTP requests.
ClientConnected(*ClientConn)
// A client connection has been closed (either by us or the client).
ClientDisconnected(*ClientConn)
// Mitmproxy has connected to a server.
ServerConnected(*ConnContext)
// A server connection has been closed (either by us or the server).
ServerDisconnected(*ConnContext)
// HTTP request headers were successfully read. At this point, the body is empty.
Requestheaders(*Flow)
// The full HTTP request has been read.
Request(*Flow)
// HTTP response headers were successfully read. At this point, the body is empty.
Responseheaders(*Flow)
// The full HTTP response has been read.
Response(*Flow)
}
// BaseAddon do nothing
type BaseAddon struct{}
func (addon *BaseAddon) ClientConnected(*ClientConn) {}
func (addon *BaseAddon) ClientDisconnected(*ClientConn) {}
func (addon *BaseAddon) ServerConnected(*ConnContext) {}
func (addon *BaseAddon) ServerDisconnected(*ConnContext) {}
func (addon *BaseAddon) Requestheaders(*Flow) {}
func (addon *BaseAddon) Request(*Flow) {}
func (addon *BaseAddon) Responseheaders(*Flow) {}
func (addon *BaseAddon) Response(*Flow) {}
// LogAddon log connection and flow
type LogAddon struct {
BaseAddon
}
func (addon *LogAddon) ClientConnected(client *ClientConn) {
log.Infof("%v client connect\n", client.Conn.RemoteAddr())
}
func (addon *LogAddon) ClientDisconnected(client *ClientConn) {
log.Infof("%v client disconnect\n", client.Conn.RemoteAddr())
}
func (addon *LogAddon) ServerConnected(connCtx *ConnContext) {
log.Infof("%v server connect %v (%v)\n", connCtx.ClientConn.Conn.RemoteAddr(), connCtx.ServerConn.Address, connCtx.ServerConn.Conn.RemoteAddr())
}
func (addon *LogAddon) ServerDisconnected(connCtx *ConnContext) {
log.Infof("%v server disconnect %v (%v)\n", connCtx.ClientConn.Conn.RemoteAddr(), connCtx.ServerConn.Address, connCtx.ServerConn.Conn.RemoteAddr())
}
func (addon *LogAddon) Requestheaders(f *Flow) {
start := time.Now()
go func() {
<-f.Done()
var StatusCode int
if f.Response != nil {
StatusCode = f.Response.StatusCode
}
var contentLen int
if f.Response != nil && f.Response.Body != nil {
contentLen = len(f.Response.Body)
}
log.Infof("%v %v %v %v %v - %v ms\n", f.ConnContext.ClientConn.Conn.RemoteAddr(), f.Request.Method, f.Request.URL.String(), StatusCode, contentLen, time.Since(start).Milliseconds())
}()
}

@ -0,0 +1,232 @@
package proxy
import (
"context"
"crypto/tls"
"net"
"net/http"
uuid "github.com/satori/go.uuid"
)
// client connection
type ClientConn struct {
Id uuid.UUID
Conn net.Conn
Tls bool
}
func newClientConn(c net.Conn) *ClientConn {
return &ClientConn{
Id: uuid.NewV4(),
Conn: c,
Tls: false,
}
}
// server connection
type ServerConn struct {
Id uuid.UUID
Address string
Conn net.Conn
client *http.Client
}
func newServerConn() *ServerConn {
return &ServerConn{
Id: uuid.NewV4(),
}
}
// connection context ctx key
var connContextKey = new(struct{})
// connection context
type ConnContext struct {
ClientConn *ClientConn
ServerConn *ServerConn
proxy *Proxy
}
func newConnContext(c net.Conn, proxy *Proxy) *ConnContext {
clientConn := newClientConn(c)
return &ConnContext{
ClientConn: clientConn,
proxy: proxy,
}
}
func (connCtx *ConnContext) InitHttpServerConn() {
if connCtx.ServerConn != nil {
return
}
if connCtx.ClientConn.Tls {
return
}
serverConn := newServerConn()
serverConn.client = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
c, err := (&net.Dialer{}).DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
cw := &wrapServerConn{
Conn: c,
proxy: connCtx.proxy,
connCtx: connCtx,
}
serverConn.Conn = cw
serverConn.Address = addr
defer func() {
for _, addon := range connCtx.proxy.Addons {
addon.ServerConnected(connCtx)
}
}()
return cw, nil
},
ForceAttemptHTTP2: false, // disable http2
DisableCompression: true, // To get the original response from the server, set Transport.DisableCompression to true.
TLSClientConfig: &tls.Config{
InsecureSkipVerify: connCtx.proxy.Opts.SslInsecure,
KeyLogWriter: getTlsKeyLogWriter(),
},
},
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// 禁止自动重定向
return http.ErrUseLastResponse
},
}
connCtx.ServerConn = serverConn
}
func (connCtx *ConnContext) InitHttpsServerConn() {
if connCtx.ServerConn != nil {
return
}
if !connCtx.ClientConn.Tls {
return
}
ServerConn := newServerConn()
ServerConn.client = &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
log.Debugln("in https DialTLSContext")
plainConn, err := (&net.Dialer{}).DialContext(ctx, network, addr)
if err != nil {
return nil, err
}
cw := &wrapServerConn{
Conn: plainConn,
proxy: connCtx.proxy,
connCtx: connCtx,
}
ServerConn.Conn = cw
ServerConn.Address = addr
for _, addon := range connCtx.proxy.Addons {
addon.ServerConnected(connCtx)
}
firstTLSHost, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
cfg := &tls.Config{
InsecureSkipVerify: connCtx.proxy.Opts.SslInsecure,
KeyLogWriter: getTlsKeyLogWriter(),
ServerName: firstTLSHost,
}
tlsConn := tls.Client(cw, cfg)
return tlsConn, nil
},
ForceAttemptHTTP2: false, // disable http2
DisableCompression: true, // To get the original response from the server, set Transport.DisableCompression to true.
},
CheckRedirect: func(req *http.Request, via []*http.Request) error {
// 禁止自动重定向
return http.ErrUseLastResponse
},
}
connCtx.ServerConn = ServerConn
}
// wrap tcpConn for remote client
type wrapClientConn struct {
net.Conn
proxy *Proxy
connCtx *ConnContext
closed bool
closeErr error
}
func (c *wrapClientConn) Close() error {
log.Debugln("in wrapClientConn close")
if c.closed {
return c.closeErr
}
c.closed = true
c.closeErr = c.Conn.Close()
for _, addon := range c.proxy.Addons {
addon.ClientDisconnected(c.connCtx.ClientConn)
}
if c.connCtx.ServerConn != nil && c.connCtx.ServerConn.Conn != nil {
c.connCtx.ServerConn.Conn.Close()
}
return c.closeErr
}
// wrap tcpListener for remote client
type wrapListener struct {
net.Listener
proxy *Proxy
}
func (l *wrapListener) Accept() (net.Conn, error) {
c, err := l.Listener.Accept()
if err != nil {
return nil, err
}
return &wrapClientConn{
Conn: c,
proxy: l.proxy,
}, nil
}
// wrap tcpConn for remote server
type wrapServerConn struct {
net.Conn
proxy *Proxy
connCtx *ConnContext
closed bool
closeErr error
}
func (c *wrapServerConn) Close() error {
log.Debugln("in wrapServerConn close")
if c.closed {
return c.closeErr
}
c.closed = true
c.closeErr = c.Conn.Close()
for _, addon := range c.proxy.Addons {
addon.ServerDisconnected(c.connCtx)
}
c.connCtx.ClientConn.Conn.Close()
return c.closeErr
}

@ -1,4 +1,4 @@
package flow package proxy
import ( import (
"encoding/json" "encoding/json"
@ -7,11 +7,9 @@ import (
"net/url" "net/url"
uuid "github.com/satori/go.uuid" uuid "github.com/satori/go.uuid"
_log "github.com/sirupsen/logrus"
) )
var log = _log.WithField("at", "flow") // flow http request
type Request struct { type Request struct {
Method string Method string
URL *url.URL URL *url.URL
@ -22,6 +20,20 @@ type Request struct {
raw *http.Request raw *http.Request
} }
func newRequest(req *http.Request) *Request {
return &Request{
Method: req.Method,
URL: req.URL,
Proto: req.Proto,
Header: req.Header,
raw: req,
}
}
func (r *Request) Raw() *http.Request {
return r.raw
}
func (req *Request) MarshalJSON() ([]byte, error) { func (req *Request) MarshalJSON() ([]byte, error) {
r := make(map[string]interface{}) r := make(map[string]interface{})
r["method"] = req.Method r["method"] = req.Method
@ -79,20 +91,7 @@ func (req *Request) UnmarshalJSON(data []byte) error {
return nil return nil
} }
func NewRequest(req *http.Request) *Request { // flow http response
return &Request{
Method: req.Method,
URL: req.URL,
Proto: req.Proto,
Header: req.Header,
raw: req,
}
}
func (r *Request) Raw() *http.Request {
return r.raw
}
type Response struct { type Response struct {
StatusCode int `json:"statusCode"` StatusCode int `json:"statusCode"`
Header http.Header `json:"header"` Header http.Header `json:"header"`
@ -103,31 +102,24 @@ type Response struct {
decodedErr error decodedErr error
} }
// flow
type Flow struct { type Flow struct {
*Request Id uuid.UUID
*Response ConnContext *ConnContext
Request *Request
Response *Response
// https://docs.mitmproxy.org/stable/overview-features/#streaming // https://docs.mitmproxy.org/stable/overview-features/#streaming
// 如果为 true则不缓冲 Request.Body 和 Response.Body且不进入之后的 Addon.Request 和 Addon.Response // 如果为 true则不缓冲 Request.Body 和 Response.Body且不进入之后的 Addon.Request 和 Addon.Response
Stream bool Stream bool
done chan struct{}
Id uuid.UUID done chan struct{}
ConnContext *ConnContext
}
func (f *Flow) MarshalJSON() ([]byte, error) {
j := make(map[string]interface{})
j["id"] = f.Id
j["request"] = f.Request
j["response"] = f.Response
return json.Marshal(j)
} }
func NewFlow() *Flow { func newFlow() *Flow {
return &Flow{ return &Flow{
done: make(chan struct{}),
Id: uuid.NewV4(), Id: uuid.NewV4(),
done: make(chan struct{}),
} }
} }
@ -135,6 +127,14 @@ func (f *Flow) Done() <-chan struct{} {
return f.done return f.done
} }
func (f *Flow) Finish() { func (f *Flow) finish() {
close(f.done) close(f.done)
} }
func (f *Flow) MarshalJSON() ([]byte, error) {
j := make(map[string]interface{})
j["id"] = f.Id
j["request"] = f.Request
j["response"] = f.Response
return json.Marshal(j)
}

@ -1,4 +1,4 @@
package flow package proxy
import ( import (
"bytes" "bytes"
@ -12,9 +12,7 @@ import (
"github.com/andybalholm/brotli" "github.com/andybalholm/brotli"
) )
// handle http header: content-encoding var errEncodingNotSupport = errors.New("content-encoding not support")
var EncodingNotSupport = errors.New("content-encoding not support")
var textContentTypes = []string{ var textContentTypes = []string{
"text", "text",
@ -59,7 +57,7 @@ func (r *Response) DecodedBody() ([]byte, error) {
return r.decodedBody, nil return r.decodedBody, nil
} }
decodedBody, decodedErr := Decode(enc, r.Body) decodedBody, decodedErr := decode(enc, r.Body)
if decodedErr != nil { if decodedErr != nil {
r.decodedErr = decodedErr r.decodedErr = decodedErr
log.Error(r.decodedErr) log.Error(r.decodedErr)
@ -83,7 +81,7 @@ func (r *Response) ReplaceToDecodedBody() {
r.Header.Del("Transfer-Encoding") r.Header.Del("Transfer-Encoding")
} }
func Decode(enc string, body []byte) ([]byte, error) { func decode(enc string, body []byte) ([]byte, error) {
if enc == "gzip" { if enc == "gzip" {
dreader, err := gzip.NewReader(bytes.NewReader(body)) dreader, err := gzip.NewReader(bytes.NewReader(body))
if err != nil { if err != nil {
@ -117,5 +115,5 @@ func Decode(enc string, body []byte) ([]byte, error) {
return buf.Bytes(), nil return buf.Bytes(), nil
} }
return nil, EncodingNotSupport return nil, errEncodingNotSupport
} }

@ -3,12 +3,14 @@ package proxy
import ( import (
"bytes" "bytes"
"io" "io"
"os"
"strings" "strings"
"sync"
_log "github.com/sirupsen/logrus" _log "github.com/sirupsen/logrus"
) )
var NormalErrMsgs []string = []string{ var normalErrMsgs []string = []string{
"read: connection reset by peer", "read: connection reset by peer",
"write: broken pipe", "write: broken pipe",
"i/o timeout", "i/o timeout",
@ -20,10 +22,10 @@ var NormalErrMsgs []string = []string{
} }
// 仅打印预料之外的错误信息 // 仅打印预料之外的错误信息
func LogErr(log *_log.Entry, err error) (loged bool) { func logErr(log *_log.Entry, err error) (loged bool) {
msg := err.Error() msg := err.Error()
for _, str := range NormalErrMsgs { for _, str := range normalErrMsgs {
if strings.Contains(msg, str) { if strings.Contains(msg, str) {
log.Debug(err) log.Debug(err)
return return
@ -61,7 +63,7 @@ func transfer(log *_log.Entry, a, b io.ReadWriteCloser) {
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
if err := <-errChan; err != nil { if err := <-errChan; err != nil {
LogErr(log, err) logErr(log, err)
return // 如果有错误,直接返回 return // 如果有错误,直接返回
} }
} }
@ -70,7 +72,7 @@ func transfer(log *_log.Entry, a, b io.ReadWriteCloser) {
// 尝试将 Reader 读取至 buffer 中 // 尝试将 Reader 读取至 buffer 中
// 如果未达到 limit则成功读取进入 buffer // 如果未达到 limit则成功读取进入 buffer
// 否则 buffer 返回 nil且返回新 Reader状态为未读取前 // 否则 buffer 返回 nil且返回新 Reader状态为未读取前
func ReaderToBuffer(r io.Reader, limit int64) ([]byte, io.Reader, error) { func readerToBuffer(r io.Reader, limit int64) ([]byte, io.Reader, error) {
buf := bytes.NewBuffer(make([]byte, 0)) buf := bytes.NewBuffer(make([]byte, 0))
lr := io.LimitReader(r, limit) lr := io.LimitReader(r, limit)
@ -88,3 +90,25 @@ func ReaderToBuffer(r io.Reader, limit int64) ([]byte, io.Reader, error) {
// 返回 buffer // 返回 buffer
return buf.Bytes(), nil, nil return buf.Bytes(), nil, nil
} }
// Wireshark 解析 https 设置
var tlsKeyLogWriter io.Writer
var tlsKeyLogOnce sync.Once
func getTlsKeyLogWriter() io.Writer {
tlsKeyLogOnce.Do(func() {
logfile := os.Getenv("SSLKEYLOGFILE")
if logfile == "" {
return
}
writer, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.WithField("in", "getTlsKeyLogWriter").Debug(err)
return
}
tlsKeyLogWriter = writer
})
return tlsKeyLogWriter
}

@ -6,7 +6,7 @@ import (
) )
// 拦截 https 流量通用接口 // 拦截 https 流量通用接口
type Interceptor interface { type interceptor interface {
// 初始化 // 初始化
Start() error Start() error
// 传入当前客户端 req // 传入当前客户端 req
@ -14,12 +14,12 @@ type Interceptor interface {
} }
// 直接转发 https 流量 // 直接转发 https 流量
type Forward struct{} type forward struct{}
func (i *Forward) Start() error { func (i *forward) Start() error {
return nil return nil
} }
func (i *Forward) Dial(req *http.Request) (net.Conn, error) { func (i *forward) Dial(req *http.Request) (net.Conn, error) {
return net.Dial("tcp", req.Host) return net.Dial("tcp", req.Host)
} }

@ -9,20 +9,10 @@ import (
"strings" "strings"
"github.com/lqqyt2423/go-mitmproxy/cert" "github.com/lqqyt2423/go-mitmproxy/cert"
"github.com/lqqyt2423/go-mitmproxy/flow"
) )
// 模拟了标准库中 server 运行,目的是仅通过当前进程内存转发 socket 数据,不需要经过 tcp 或 unix socket // 模拟了标准库中 server 运行,目的是仅通过当前进程内存转发 socket 数据,不需要经过 tcp 或 unix socket
// mock net.Listener
type middleListener struct {
connChan chan net.Conn
}
func (l *middleListener) Accept() (net.Conn, error) { return <-l.connChan, nil }
func (l *middleListener) Close() error { return nil }
func (l *middleListener) Addr() net.Addr { return nil }
type pipeAddr struct { type pipeAddr struct {
remoteAddr string remoteAddr string
} }
@ -30,20 +20,13 @@ type pipeAddr struct {
func (pipeAddr) Network() string { return "pipe" } func (pipeAddr) Network() string { return "pipe" }
func (a *pipeAddr) String() string { return a.remoteAddr } func (a *pipeAddr) String() string { return a.remoteAddr }
// 建立客户端和服务端通信的通道
func newPipes(req *http.Request) (net.Conn, *pipeConn) {
client, srv := net.Pipe()
server := newPipeConn(srv, req)
return client, server
}
// add Peek method for conn // add Peek method for conn
type pipeConn struct { type pipeConn struct {
net.Conn net.Conn
r *bufio.Reader r *bufio.Reader
host string host string
remoteAddr string remoteAddr string
connContext *flow.ConnContext connContext *ConnContext
} }
func newPipeConn(c net.Conn, req *http.Request) *pipeConn { func newPipeConn(c net.Conn, req *http.Request) *pipeConn {
@ -52,7 +35,7 @@ func newPipeConn(c net.Conn, req *http.Request) *pipeConn {
r: bufio.NewReader(c), r: bufio.NewReader(c),
host: req.Host, host: req.Host,
remoteAddr: req.RemoteAddr, remoteAddr: req.RemoteAddr,
connContext: req.Context().Value(flow.ConnContextKey).(*flow.ConnContext), connContext: req.Context().Value(connContextKey).(*ConnContext),
} }
} }
@ -68,33 +51,49 @@ func (c *pipeConn) RemoteAddr() net.Addr {
return &pipeAddr{remoteAddr: c.remoteAddr} return &pipeAddr{remoteAddr: c.remoteAddr}
} }
// Middle: man-in-the-middle // 建立客户端和服务端通信的通道
type Middle struct { func newPipes(req *http.Request) (net.Conn, *pipeConn) {
Proxy *Proxy client, srv := net.Pipe()
CA *cert.CA server := newPipeConn(srv, req)
Listener net.Listener return client, server
Server *http.Server }
// mock net.Listener
type middleListener struct {
connChan chan net.Conn
}
func (l *middleListener) Accept() (net.Conn, error) { return <-l.connChan, nil }
func (l *middleListener) Close() error { return nil }
func (l *middleListener) Addr() net.Addr { return nil }
// middle: man-in-the-middle server
type middle struct {
proxy *Proxy
ca *cert.CA
listener *middleListener
server *http.Server
} }
func NewMiddle(proxy *Proxy, caPath string) (Interceptor, error) { func newMiddle(proxy *Proxy) (interceptor, error) {
ca, err := cert.NewCA(caPath) ca, err := cert.NewCA(proxy.Opts.CaRootPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
m := &Middle{ m := &middle{
Proxy: proxy, proxy: proxy,
CA: ca, ca: ca,
listener: &middleListener{
connChan: make(chan net.Conn),
},
} }
server := &http.Server{ server := &http.Server{
Handler: m, Handler: m,
// IdleTimeout: 5 * time.Second,
ConnContext: func(ctx context.Context, c net.Conn) context.Context { ConnContext: func(ctx context.Context, c net.Conn) context.Context {
return context.WithValue(ctx, flow.ConnContextKey, c.(*tls.Conn).NetConn().(*pipeConn).connContext) return context.WithValue(ctx, connContextKey, c.(*tls.Conn).NetConn().(*pipeConn).connContext)
}, },
TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), // disable http2 TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), // disable http2
TLSConfig: &tls.Config{ TLSConfig: &tls.Config{
GetCertificate: func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) { GetCertificate: func(chi *tls.ClientHelloInfo) (*tls.Certificate, error) {
@ -103,28 +102,25 @@ func NewMiddle(proxy *Proxy, caPath string) (Interceptor, error) {
}, },
}, },
} }
m.server = server
m.Server = server
m.Listener = &middleListener{make(chan net.Conn)}
return m, nil return m, nil
} }
func (m *Middle) Start() error { func (m *middle) Start() error {
return m.Server.ServeTLS(m.Listener, "", "") return m.server.ServeTLS(m.listener, "", "")
} }
// todo: should block until ServerConnected // todo: should block until ServerConnected
func (m *Middle) Dial(req *http.Request) (net.Conn, error) { func (m *middle) Dial(req *http.Request) (net.Conn, error) {
pipeClientConn, pipeServerConn := newPipes(req) pipeClientConn, pipeServerConn := newPipes(req)
go m.intercept(pipeServerConn) go m.intercept(pipeServerConn)
return pipeClientConn, nil return pipeClientConn, nil
} }
func (m *Middle) ServeHTTP(res http.ResponseWriter, req *http.Request) { func (m *middle) ServeHTTP(res http.ResponseWriter, req *http.Request) {
if strings.EqualFold(req.Header.Get("Connection"), "Upgrade") && strings.EqualFold(req.Header.Get("Upgrade"), "websocket") { if strings.EqualFold(req.Header.Get("Connection"), "Upgrade") && strings.EqualFold(req.Header.Get("Upgrade"), "websocket") {
// wss // wss
DefaultWebSocket.WSS(res, req) defaultWebSocket.wss(res, req)
return return
} }
@ -134,14 +130,14 @@ func (m *Middle) ServeHTTP(res http.ResponseWriter, req *http.Request) {
if req.URL.Host == "" { if req.URL.Host == "" {
req.URL.Host = req.Host req.URL.Host = req.Host
} }
m.Proxy.ServeHTTP(res, req) m.proxy.ServeHTTP(res, req)
} }
// 解析 connect 流量 // 解析 connect 流量
// 如果是 tls 流量,则进入 listener.Accept => Middle.ServeHTTP // 如果是 tls 流量,则进入 listener.Accept => Middle.ServeHTTP
// 否则很可能是 ws 流量 // 否则很可能是 ws 流量
func (m *Middle) intercept(pipeServerConn *pipeConn) { func (m *middle) intercept(pipeServerConn *pipeConn) {
log := log.WithField("in", "Middle.intercept").WithField("host", pipeServerConn.host) log := log.WithField("in", "middle.intercept").WithField("host", pipeServerConn.host)
buf, err := pipeServerConn.Peek(3) buf, err := pipeServerConn.Peek(3)
if err != nil { if err != nil {
@ -153,25 +149,11 @@ func (m *Middle) intercept(pipeServerConn *pipeConn) {
// https://github.com/mitmproxy/mitmproxy/blob/main/mitmproxy/net/tls.py is_tls_record_magic // https://github.com/mitmproxy/mitmproxy/blob/main/mitmproxy/net/tls.py is_tls_record_magic
if buf[0] == 0x16 && buf[1] == 0x03 && buf[2] <= 0x03 { if buf[0] == 0x16 && buf[1] == 0x03 && buf[2] <= 0x03 {
// tls // tls
pipeServerConn.connContext.Client.Tls = true pipeServerConn.connContext.ClientConn.Tls = true
pipeServerConn.connContext.InitHttpsServer( pipeServerConn.connContext.InitHttpsServerConn()
m.Proxy.Opts.SslInsecure, m.listener.connChan <- pipeServerConn
func(c net.Conn) net.Conn {
return &serverConn{
Conn: c,
proxy: m.Proxy,
connCtx: pipeServerConn.connContext,
}
},
func() {
for _, addon := range m.Proxy.Addons {
addon.ServerConnected(pipeServerConn.connContext)
}
},
)
m.Listener.(*middleListener).connChan <- pipeServerConn
} else { } else {
// ws // ws
DefaultWebSocket.WS(pipeServerConn, pipeServerConn.host) defaultWebSocket.ws(pipeServerConn, pipeServerConn.host)
} }
} }

@ -7,8 +7,6 @@ import (
"net" "net"
"net/http" "net/http"
"github.com/lqqyt2423/go-mitmproxy/addon"
"github.com/lqqyt2423/go-mitmproxy/flow"
_log "github.com/sirupsen/logrus" _log "github.com/sirupsen/logrus"
) )
@ -25,118 +23,46 @@ type Options struct {
type Proxy struct { type Proxy struct {
Opts *Options Opts *Options
Version string Version string
Server *http.Server Addons []Addon
Interceptor Interceptor
Addons []addon.Addon
}
type proxyListener struct { server *http.Server
net.Listener interceptor interceptor
proxy *Proxy
} }
func (l *proxyListener) Accept() (net.Conn, error) { func NewProxy(opts *Options) (*Proxy, error) {
c, err := l.Listener.Accept() if opts.StreamLargeBodies <= 0 {
if err != nil { opts.StreamLargeBodies = 1024 * 1024 * 5 // default: 5mb
return nil, err
}
return &proxyConn{
Conn: c,
proxy: l.proxy,
}, nil
}
type proxyConn struct {
net.Conn
proxy *Proxy
connCtx *flow.ConnContext
closed bool
closeErr error
}
func (c *proxyConn) Close() error {
log.Debugln("in proxyConn close")
if c.closed {
return c.closeErr
}
c.closed = true
c.closeErr = c.Conn.Close()
for _, addon := range c.proxy.Addons {
addon.ClientDisconnected(c.connCtx.Client)
}
if c.connCtx.Server != nil && c.connCtx.Server.Conn != nil {
c.connCtx.Server.Conn.Close()
}
return c.closeErr
}
type serverConn struct {
net.Conn
proxy *Proxy
connCtx *flow.ConnContext
closed bool
closeErr error
}
func (c *serverConn) Close() error {
log.Debugln("in http serverConn close")
if c.closed {
return c.closeErr
} }
c.closed = true proxy := &Proxy{
c.closeErr = c.Conn.Close() Opts: opts,
Version: "1.0.0",
for _, addon := range c.proxy.Addons { Addons: make([]Addon, 0),
addon.ServerDisconnected(c.connCtx)
} }
c.connCtx.Client.Conn.Close() proxy.server = &http.Server{
return c.closeErr
}
func NewProxy(opts *Options) (*Proxy, error) {
proxy := new(Proxy)
proxy.Opts = opts
proxy.Version = "0.2.0"
proxy.Server = &http.Server{
Addr: opts.Addr, Addr: opts.Addr,
Handler: proxy, Handler: proxy,
// IdleTimeout: 5 * time.Second,
ConnContext: func(ctx context.Context, c net.Conn) context.Context { ConnContext: func(ctx context.Context, c net.Conn) context.Context {
connCtx := flow.NewConnContext(c) connCtx := newConnContext(c, proxy)
for _, addon := range proxy.Addons { for _, addon := range proxy.Addons {
addon.ClientConnected(connCtx.Client) addon.ClientConnected(connCtx.ClientConn)
} }
c.(*proxyConn).connCtx = connCtx c.(*wrapClientConn).connCtx = connCtx
return context.WithValue(ctx, flow.ConnContextKey, connCtx) return context.WithValue(ctx, connContextKey, connCtx)
}, },
} }
interceptor, err := NewMiddle(proxy, opts.CaRootPath) interceptor, err := newMiddle(proxy)
if err != nil { if err != nil {
return nil, err return nil, err
} }
proxy.Interceptor = interceptor proxy.interceptor = interceptor
if opts.StreamLargeBodies <= 0 {
opts.StreamLargeBodies = 1024 * 1024 * 5 // default: 5mb
}
proxy.Addons = make([]addon.Addon, 0)
return proxy, nil return proxy, nil
} }
func (proxy *Proxy) AddAddon(addon addon.Addon) { func (proxy *Proxy) AddAddon(addon Addon) {
proxy.Addons = append(proxy.Addons, addon) proxy.Addons = append(proxy.Addons, addon)
} }
@ -144,8 +70,8 @@ func (proxy *Proxy) Start() error {
errChan := make(chan error) errChan := make(chan error)
go func() { go func() {
log.Infof("Proxy start listen at %v\n", proxy.Server.Addr) log.Infof("Proxy start listen at %v\n", proxy.server.Addr)
addr := proxy.Server.Addr addr := proxy.server.Addr
if addr == "" { if addr == "" {
addr = ":http" addr = ":http"
} }
@ -154,16 +80,16 @@ func (proxy *Proxy) Start() error {
errChan <- err errChan <- err
return return
} }
pln := &proxyListener{ pln := &wrapListener{
Listener: ln, Listener: ln,
proxy: proxy, proxy: proxy,
} }
err = proxy.Server.Serve(pln) err = proxy.server.Serve(pln)
errChan <- err errChan <- err
}() }()
go func() { go func() {
err := proxy.Interceptor.Start() err := proxy.interceptor.Start()
errChan <- err errChan <- err
}() }()
@ -194,7 +120,7 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) {
return return
} }
reply := func(response *flow.Response, body io.Reader) { reply := func(response *Response, body io.Reader) {
if response.Header != nil { if response.Header != nil {
for key, value := range response.Header { for key, value := range response.Header {
for _, v := range value { for _, v := range value {
@ -207,12 +133,12 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) {
if body != nil { if body != nil {
_, err := io.Copy(res, body) _, err := io.Copy(res, body)
if err != nil { if err != nil {
LogErr(log, err) logErr(log, err)
} }
} else if response.Body != nil && len(response.Body) > 0 { } else if response.Body != nil && len(response.Body) > 0 {
_, err := res.Write(response.Body) _, err := res.Write(response.Body)
if err != nil { if err != nil {
LogErr(log, err) logErr(log, err)
} }
} }
} }
@ -224,10 +150,10 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) {
} }
}() }()
f := flow.NewFlow() f := newFlow()
f.Request = flow.NewRequest(req) f.Request = newRequest(req)
f.ConnContext = req.Context().Value(flow.ConnContextKey).(*flow.ConnContext) f.ConnContext = req.Context().Value(connContextKey).(*ConnContext)
defer f.Finish() defer f.finish()
// trigger addon event Requestheaders // trigger addon event Requestheaders
for _, addon := range proxy.Addons { for _, addon := range proxy.Addons {
@ -241,7 +167,7 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) {
// Read request body // Read request body
var reqBody io.Reader = req.Body var reqBody io.Reader = req.Body
if !f.Stream { if !f.Stream {
reqBuf, r, err := ReaderToBuffer(req.Body, proxy.Opts.StreamLargeBodies) reqBuf, r, err := readerToBuffer(req.Body, proxy.Opts.StreamLargeBodies)
reqBody = r reqBody = r
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -280,31 +206,16 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) {
} }
} }
f.ConnContext.InitHttpServer( f.ConnContext.InitHttpServerConn()
proxy.Opts.SslInsecure, proxyRes, err := f.ConnContext.ServerConn.client.Do(proxyReq)
func(c net.Conn) net.Conn {
return &serverConn{
Conn: c,
proxy: proxy,
connCtx: f.ConnContext,
}
},
func() {
for _, addon := range proxy.Addons {
addon.ServerConnected(f.ConnContext)
}
},
)
proxyRes, err := f.ConnContext.Server.Client.Do(proxyReq)
if err != nil { if err != nil {
LogErr(log, err) logErr(log, err)
res.WriteHeader(502) res.WriteHeader(502)
return return
} }
defer proxyRes.Body.Close() defer proxyRes.Body.Close()
f.Response = &flow.Response{ f.Response = &Response{
StatusCode: proxyRes.StatusCode, StatusCode: proxyRes.StatusCode,
Header: proxyRes.Header, Header: proxyRes.Header,
} }
@ -321,7 +232,7 @@ func (proxy *Proxy) ServeHTTP(res http.ResponseWriter, req *http.Request) {
// Read response body // Read response body
var resBody io.Reader = proxyRes.Body var resBody io.Reader = proxyRes.Body
if !f.Stream { if !f.Stream {
resBuf, r, err := ReaderToBuffer(proxyRes.Body, proxy.Opts.StreamLargeBodies) resBuf, r, err := readerToBuffer(proxyRes.Body, proxy.Opts.StreamLargeBodies)
resBody = r resBody = r
if err != nil { if err != nil {
log.Error(err) log.Error(err)
@ -352,7 +263,7 @@ func (proxy *Proxy) handleConnect(res http.ResponseWriter, req *http.Request) {
log.Debug("receive connect") log.Debug("receive connect")
conn, err := proxy.Interceptor.Dial(req) conn, err := proxy.interceptor.Dial(req)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
res.WriteHeader(502) res.WriteHeader(502)

@ -10,25 +10,25 @@ import (
// 当前仅做了转发 websocket 流量 // 当前仅做了转发 websocket 流量
type WebSocket struct{} type webSocket struct{}
var DefaultWebSocket WebSocket var defaultWebSocket webSocket
func (s *WebSocket) WS(conn net.Conn, host string) { func (s *webSocket) ws(conn net.Conn, host string) {
log := log.WithField("in", "WebSocket.WS").WithField("host", host) log := log.WithField("in", "webSocket.ws").WithField("host", host)
defer conn.Close() defer conn.Close()
remoteConn, err := net.Dial("tcp", host) remoteConn, err := net.Dial("tcp", host)
if err != nil { if err != nil {
LogErr(log, err) logErr(log, err)
return return
} }
defer remoteConn.Close() defer remoteConn.Close()
transfer(log, conn, remoteConn) transfer(log, conn, remoteConn)
} }
func (s *WebSocket) WSS(res http.ResponseWriter, req *http.Request) { func (s *webSocket) wss(res http.ResponseWriter, req *http.Request) {
log := log.WithField("in", "WebSocket.WSS").WithField("host", req.Host) log := log.WithField("in", "webSocket.wss").WithField("host", req.Host)
upgradeBuf, err := httputil.DumpRequest(req, false) upgradeBuf, err := httputil.DumpRequest(req, false)
if err != nil { if err != nil {

Before

Width:  |  Height:  |  Size: 3.8 KiB

After

Width:  |  Height:  |  Size: 3.8 KiB

Before

Width:  |  Height:  |  Size: 5.2 KiB

After

Width:  |  Height:  |  Size: 5.2 KiB

Before

Width:  |  Height:  |  Size: 9.4 KiB

After

Width:  |  Height:  |  Size: 9.4 KiB

Before

Width:  |  Height:  |  Size: 3.8 KiB

After

Width:  |  Height:  |  Size: 3.8 KiB

Before

Width:  |  Height:  |  Size: 5.2 KiB

After

Width:  |  Height:  |  Size: 5.2 KiB

Before

Width:  |  Height:  |  Size: 9.4 KiB

After

Width:  |  Height:  |  Size: 9.4 KiB

@ -5,7 +5,7 @@ import (
"sync" "sync"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
"github.com/lqqyt2423/go-mitmproxy/flow" "github.com/lqqyt2423/go-mitmproxy/proxy"
) )
type breakPointRule struct { type breakPointRule struct {
@ -31,7 +31,7 @@ func newConn(c *websocket.Conn) *concurrentConn {
} }
} }
func (c *concurrentConn) writeMessage(msg *messageFlow, f *flow.Flow) { func (c *concurrentConn) writeMessage(msg *messageFlow, f *proxy.Flow) {
if c.isIntercpt(f, msg) { if c.isIntercpt(f, msg) {
msg.waitIntercept = 1 msg.waitIntercept = 1
} }
@ -94,7 +94,7 @@ func (c *concurrentConn) initWaitChan(key string) chan interface{} {
} }
// 是否拦截 // 是否拦截
func (c *concurrentConn) isIntercpt(f *flow.Flow, after *messageFlow) bool { func (c *concurrentConn) isIntercpt(f *proxy.Flow, after *messageFlow) bool {
if after.mType != messageTypeRequestBody && after.mType != messageTypeResponseBody { if after.mType != messageTypeRequestBody && after.mType != messageTypeResponseBody {
return false return false
} }
@ -129,13 +129,13 @@ func (c *concurrentConn) isIntercpt(f *flow.Flow, after *messageFlow) bool {
} }
// 拦截 // 拦截
func (c *concurrentConn) waitIntercept(f *flow.Flow, after *messageFlow) { func (c *concurrentConn) waitIntercept(f *proxy.Flow, after *messageFlow) {
ch := c.initWaitChan(f.Id.String()) ch := c.initWaitChan(f.Id.String())
msg := (<-ch).(*messageEdit) msg := (<-ch).(*messageEdit)
// drop // drop
if msg.mType == messageTypeDropRequest || msg.mType == messageTypeDropResponse { if msg.mType == messageTypeDropRequest || msg.mType == messageTypeDropResponse {
f.Response = &flow.Response{ f.Response = &proxy.Response{
StatusCode: 502, StatusCode: 502,
} }
return return

@ -6,7 +6,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"github.com/lqqyt2423/go-mitmproxy/flow" "github.com/lqqyt2423/go-mitmproxy/proxy"
uuid "github.com/satori/go.uuid" uuid "github.com/satori/go.uuid"
) )
@ -74,7 +74,7 @@ type messageFlow struct {
content []byte content []byte
} }
func newMessageFlow(mType messageType, f *flow.Flow) *messageFlow { func newMessageFlow(mType messageType, f *proxy.Flow) *messageFlow {
var content []byte var content []byte
var err error = nil var err error = nil
@ -114,8 +114,8 @@ func (m *messageFlow) bytes() []byte {
type messageEdit struct { type messageEdit struct {
mType messageType mType messageType
id uuid.UUID id uuid.UUID
request *flow.Request request *proxy.Request
response *flow.Response response *proxy.Response
} }
func parseMessageEdit(data []byte) *messageEdit { func parseMessageEdit(data []byte) *messageEdit {
@ -158,7 +158,7 @@ func parseMessageEdit(data []byte) *messageEdit {
bodyContent := data[42+hl+4:] bodyContent := data[42+hl+4:]
if mType == messageTypeChangeRequest { if mType == messageTypeChangeRequest {
req := new(flow.Request) req := new(proxy.Request)
err := json.Unmarshal(headerContent, req) err := json.Unmarshal(headerContent, req)
if err != nil { if err != nil {
return nil return nil
@ -166,7 +166,7 @@ func parseMessageEdit(data []byte) *messageEdit {
req.Body = bodyContent req.Body = bodyContent
msg.request = req msg.request = req
} else if mType == messageTypeChangeResponse { } else if mType == messageTypeChangeResponse {
res := new(flow.Response) res := new(proxy.Response)
err := json.Unmarshal(headerContent, res) err := json.Unmarshal(headerContent, res)
if err != nil { if err != nil {
return nil return nil

@ -7,8 +7,7 @@ import (
"sync" "sync"
"github.com/gorilla/websocket" "github.com/gorilla/websocket"
"github.com/lqqyt2423/go-mitmproxy/addon" "github.com/lqqyt2423/go-mitmproxy/proxy"
"github.com/lqqyt2423/go-mitmproxy/flow"
_log "github.com/sirupsen/logrus" _log "github.com/sirupsen/logrus"
) )
@ -17,25 +16,8 @@ var log = _log.WithField("at", "web addon")
//go:embed client/build //go:embed client/build
var assets embed.FS var assets embed.FS
func (web *WebAddon) echo(w http.ResponseWriter, r *http.Request) {
c, err := web.upgrader.Upgrade(w, r, nil)
if err != nil {
log.Print("upgrade:", err)
return
}
conn := newConn(c)
web.addConn(conn)
defer func() {
web.removeConn(conn)
c.Close()
}()
conn.readloop()
}
type WebAddon struct { type WebAddon struct {
addon.Base proxy.BaseAddon
upgrader *websocket.Upgrader upgrader *websocket.Upgrader
conns []*concurrentConn conns []*concurrentConn
@ -72,6 +54,23 @@ func NewWebAddon(addr string) *WebAddon {
return web return web
} }
func (web *WebAddon) echo(w http.ResponseWriter, r *http.Request) {
c, err := web.upgrader.Upgrade(w, r, nil)
if err != nil {
log.Print("upgrade:", err)
return
}
conn := newConn(c)
web.addConn(conn)
defer func() {
web.removeConn(conn)
c.Close()
}()
conn.readloop()
}
func (web *WebAddon) addConn(c *concurrentConn) { func (web *WebAddon) addConn(c *concurrentConn) {
web.connsMu.Lock() web.connsMu.Lock()
web.conns = append(web.conns, c) web.conns = append(web.conns, c)
@ -96,7 +95,7 @@ func (web *WebAddon) removeConn(conn *concurrentConn) {
web.conns = append(web.conns[:index], web.conns[index+1:]...) web.conns = append(web.conns[:index], web.conns[index+1:]...)
} }
func (web *WebAddon) sendFlow(f *flow.Flow, msgFn func() *messageFlow) bool { func (web *WebAddon) sendFlow(f *proxy.Flow, msgFn func() *messageFlow) bool {
web.connsMu.RLock() web.connsMu.RLock()
conns := web.conns conns := web.conns
web.connsMu.RUnlock() web.connsMu.RUnlock()
@ -113,25 +112,25 @@ func (web *WebAddon) sendFlow(f *flow.Flow, msgFn func() *messageFlow) bool {
return true return true
} }
func (web *WebAddon) Requestheaders(f *flow.Flow) { func (web *WebAddon) Requestheaders(f *proxy.Flow) {
web.sendFlow(f, func() *messageFlow { web.sendFlow(f, func() *messageFlow {
return newMessageFlow(messageTypeRequest, f) return newMessageFlow(messageTypeRequest, f)
}) })
} }
func (web *WebAddon) Request(f *flow.Flow) { func (web *WebAddon) Request(f *proxy.Flow) {
web.sendFlow(f, func() *messageFlow { web.sendFlow(f, func() *messageFlow {
return newMessageFlow(messageTypeRequestBody, f) return newMessageFlow(messageTypeRequestBody, f)
}) })
} }
func (web *WebAddon) Responseheaders(f *flow.Flow) { func (web *WebAddon) Responseheaders(f *proxy.Flow) {
web.sendFlow(f, func() *messageFlow { web.sendFlow(f, func() *messageFlow {
return newMessageFlow(messageTypeResponse, f) return newMessageFlow(messageTypeResponse, f)
}) })
} }
func (web *WebAddon) Response(f *flow.Flow) { func (web *WebAddon) Response(f *proxy.Flow) {
web.sendFlow(f, func() *messageFlow { web.sendFlow(f, func() *messageFlow {
return newMessageFlow(messageTypeResponseBody, f) return newMessageFlow(messageTypeResponseBody, f)
}) })
Loading…
Cancel
Save