Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
name: CI

on:
pull_request:
push:
branches: [main]

permissions:
contents: read

jobs:
linux:
name: Linux / Go ${{ matrix.go }}
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
go: ['1.26.10', '1.27.3']
env:
GOTOOLCHAIN: local
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
- uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
with:
go-version: ${{ matrix.go }}
- name: Verify modules
run: go mod verify
- name: Test with race detection
run: go test -race -count=1 -timeout=5m ./...
- name: Test legacy HTTP/2 transport with race detection
run: go test -race -tags=http2legacy -count=1 -timeout=5m ./...
- name: Vet
run: go vet ./...
- name: Build Linux binary
run: CGO_ENABLED=0 go build -o "$RUNNER_TEMP/cq-source-ipgeolocation" ./
- name: Check for reachable vulnerabilities
run: |
go install golang.org/x/vuln/cmd/govulncheck@v1.9.0
govulncheck ./...
17 changes: 14 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ Column by column reference: [`docs/tables.md`](docs/tables.md).
- Go 1.26.5 or later, to build the plugin binary
- An IPGeolocation.io API key. The free Developer plan gives you 1,000 requests a day and covers the geolocation table. Sign up at [app.ipgeolocation.io/signup](https://app.ipgeolocation.io/signup).

Build release binaries with a patched Go toolchain: Go 1.26.10 or later on the
1.26 line, or Go 1.27.3 or later. Updating modules does not patch the standard
library embedded in an existing binary; rebuild and replace deployed binaries.

## Quick Start

```bash
Expand Down Expand Up @@ -118,12 +122,19 @@ The `example/` directory has eight configs you can run as is.
## Development

```bash
go build ./... # compile
go test ./... -v # unit tests, no API key needed
go build ./...
go test -race -count=1 ./...
go test -race -tags=http2legacy -count=1 ./...
go vet ./...
```

The tests use a stub HTTP server, so they run offline and make no API calls.
Tests use local HTTP servers and need no API key. The plugin sync integration test
runs all five tables through configuration, scheduling and Arrow row emission,
including optional modules, table filtering and caller-IP lookup. The legacy
transport run also exercises the x/net HTTP/2 implementation on Go 1.27.

Linux CI runs both test modes, module verification, vet, a build and govulncheck
on Go 1.26.10 and 1.27.3.

## License

Expand Down
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/cloudquery/plugin-sdk/v4 v4.96.3
github.com/rs/zerolog v1.35.1
github.com/stretchr/testify v1.12.1
golang.org/x/net v0.60.0
golang.org/x/time v0.8.0
)

Expand Down Expand Up @@ -80,12 +81,11 @@ require (
go.yaml.in/yaml/v3 v3.0.5 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect
golang.org/x/mod v0.39.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/mod v0.41.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 // indirect
golang.org/x/text v0.41.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/tools v0.49.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
Expand Down
20 changes: 10 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -201,19 +201,19 @@ go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s=
go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk=
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q=
golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
Expand Down
150 changes: 150 additions & 0 deletions internal/ipgeolocation/http2_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
package ipgeolocation

import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"io"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/http2"
"golang.org/x/net/http2/hpack"
)

func TestClientHTTP2Requests(t *testing.T) {
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, 2, r.ProtoMajor)
assert.Equal(t, "test-key", r.URL.Query().Get("apiKey"))
assert.Equal(t, "cq-http2-test", r.UserAgent())
assert.Equal(t, "application/json", r.Header.Get("Accept"))
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/v3/ipgeo":
assert.Equal(t, http.MethodGet, r.Method)
assert.Equal(t, "1.1.1.1", r.URL.Query().Get("ip"))
assert.Equal(t, "security,abuse", r.URL.Query().Get("include"))
_, _ = w.Write([]byte(`{"ip":"1.1.1.1","location":{"country_name":"Australia"}}`))
case "/v3/user-agent":
assert.Equal(t, http.MethodPost, r.Method)
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
var body map[string]string
assert.NoError(t, json.NewDecoder(r.Body).Decode(&body))
assert.Equal(t, map[string]string{"uaString": "test-ua"}, body)
_, _ = w.Write([]byte(`{"user_agent_string":"test-ua","name":"TestBrowser"}`))
default:
t.Errorf("unexpected request: %s", r.URL.Path)
w.WriteHeader(http.StatusNotFound)
}
}))
server.EnableHTTP2 = true
server.StartTLS()
t.Cleanup(server.Close)

transport := &http2.Transport{
TLSClientConfig: server.Client().Transport.(*http.Transport).TLSClientConfig.Clone(),
}
t.Cleanup(transport.CloseIdleConnections)
c, err := NewClient("test-key",
WithEndpoint(server.URL),
WithHTTPClient(&http.Client{Transport: transport, Timeout: 5 * time.Second}),
WithUserAgent("cq-http2-test"),
WithRetries(0),
)
require.NoError(t, err)

geo, err := c.GetIPGeolocation(context.Background(), "1.1.1.1", "security", "abuse")
require.NoError(t, err)
assert.Equal(t, "1.1.1.1", geo.IP)
assert.Equal(t, "Australia", geo.Location.CountryName)

ua, err := c.GetUserAgent(context.Background(), "test-ua")
require.NoError(t, err)
assert.Equal(t, "test-ua", ua.UserAgentString)
assert.Equal(t, "TestBrowser", ua.Name)
}

func TestHTTP2ResponseFraming(t *testing.T) {
for _, tc := range []struct {
name string
lengths []string
wantLengths []string
}{
{name: "valid", lengths: []string{"2"}, wantLengths: []string{"2"}},
{name: "conflicting", lengths: []string{"2", "1"}},
} {
t.Run(tc.name, func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
serverDone := make(chan error, 1)
go func() {
serverDone <- func() error {
conn, err := listener.Accept()
if err != nil {
return err
}
defer conn.Close()
if err := conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil {
return err
}
preface := make([]byte, len(http2.ClientPreface))
if _, err := io.ReadFull(conn, preface); err != nil {
return err
}
framer := http2.NewFramer(conn, conn)
if err := framer.WriteSettings(); err != nil {
return err
}
for {
frame, err := framer.ReadFrame()
if err != nil {
return err
}
if headers, ok := frame.(*http2.HeadersFrame); ok {
// Write raw fields: net/http sanitizes duplicate lengths before sending them.
var block bytes.Buffer
encoder := hpack.NewEncoder(&block)
if err := encoder.WriteField(hpack.HeaderField{Name: ":status", Value: "200"}); err != nil {
return err
}
for _, value := range tc.lengths {
if err := encoder.WriteField(hpack.HeaderField{Name: "content-length", Value: value}); err != nil {
return err
}
}
if err := framer.WriteHeaders(http2.HeadersFrameParam{
StreamID: headers.StreamID, EndHeaders: true, BlockFragment: block.Bytes(),
}); err != nil {
return err
}
return framer.WriteData(headers.StreamID, true, []byte("{}"))
}
}
}()
}()
transport := &http2.Transport{
AllowHTTP: true,
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, network, addr)
},
}
t.Cleanup(transport.CloseIdleConnections)
client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
resp, err := client.Get("http://" + listener.Addr().String())
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.Equal(t, "{}", string(body))
assert.Equal(t, tc.wantLengths, resp.Header.Values("Content-Length"))
require.NoError(t, <-serverDone)
})
}
}
Loading
Loading