hanzo/world
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(world/insider): decode the ISO-8859-1 SEC feed (was silently empty)

Browse Source 
The live EDGAR getcurrent atom feed is served as ISO-8859-1, not UTF-8, so
plain xml.Unmarshal errored ('encoding "ISO-8859-1" declared but
Decoder.CharsetReader is nil') and parseEdgarAtom returned zero entries — the
endpoint answered 200 with count:0 in prod. Give the decoder a CharsetReader
(golang.org/x/net/html/charset) so any declared charset decodes. The unit test
now uses an ISO-8859-1 fixture with a Latin-1 byte as a regression guard.
Proven against the live feed: 100 entries decode cleanly.
This commit is contained in:
Hanzo Dev
2026-07-21 17:14:46 -07:00
parent 2ea66547f0
commit 40fcaf5a07
4 changed files with 37 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+2
View File
@@ -6,6 +6,7 @@ require (
github.com/alicebob/miniredis/v2 v2.38.0
github.com/hanzoai/sqlite v0.3.2
github.com/redis/go-redis/v9 v9.20.0
golang.org/x/net v0.54.0
)
require (
@@ -25,4 +26,5 @@ require (
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/crypto v0.52.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/text v0.37.0 // indirect
)
go.sum
+4
View File
@@ -40,7 +40,11 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
internal/world/handlers_insider.go
+10 -3
View File
@@ -1,11 +1,14 @@
package world
import (
"bytes"
"context"
"encoding/xml"
"net/http"
"strings"
"time"
"golang.org/x/net/html/charset"
)
// Insider activity from SEC EDGAR: the live feed of Form 4 filings (an insider's
@@ -80,11 +83,15 @@ type atomFeed struct {
} `xml:"entry"`
}
// parseEdgarAtom pulls entries out of an EDGAR getcurrent atom feed. Malformed
// input yields an empty slice, never a panic.
// parseEdgarAtom pulls entries out of an EDGAR getcurrent atom feed. The live
// feed is served as ISO-8859-1 (not UTF-8), so the decoder carries a
// CharsetReader — plain xml.Unmarshal errors out on any declared non-UTF-8
// charset. Malformed input yields an empty slice, never a panic.
func parseEdgarAtom(body []byte) []edgarEntry {
var f atomFeed
if err := xml.Unmarshal(body, &f); err != nil {
dec := xml.NewDecoder(bytes.NewReader(body))
dec.CharsetReader = charset.NewReaderLabel
if err := dec.Decode(&f); err != nil {
return nil
}
out := make([]edgarEntry, 0, len(f.Entries))
internal/world/handlers_insider_test.go
+21 -15
View File
@@ -36,22 +36,28 @@ func TestParseEdgarAtomMalformed(t *testing.T) {
}
func TestParseEdgarAtomWellFormed(t *testing.T) {
body := []byte(`<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<entry>
<title>4 - ACME CORP (0001234567) (Issuer)</title>
<updated>2026-07-21T14:30:00-04:00</updated>
<link href="https://www.sec.gov/Archives/edgar/data/1234567/x.htm"/>
</entry>
<entry>
<title>4 - Smith John A (0009876543) (Reporting)</title>
<updated>2026-07-21T14:25:00-04:00</updated>
<link href="https://www.sec.gov/Archives/edgar/data/9876543/y.htm"/>
</entry>
</feed>`)
// The LIVE SEC feed is served as ISO-8859-1, not UTF-8 — the byte 0xE9 below
// is a Latin-1 'é'. Plain xml.Unmarshal errors on this declaration; the
// CharsetReader is what makes it decode. This fixture guards that path.
body := []byte("<?xml version=\"1.0\" encoding=\"ISO-8859-1\" ?>\n" +
"<feed xmlns=\"http://www.w3.org/2005/Atom\">\n" +
" <entry>\n" +
" <title>4 - ACME CORP (0001234567) (Issuer)</title>\n" +
" <updated>2026-07-21T14:30:00-04:00</updated>\n" +
" <link href=\"https://www.sec.gov/Archives/edgar/data/1234567/x.htm\"/>\n" +
" </entry>\n" +
" <entry>\n" +
" <title>4 - Beaumont\xe9 Corp (0009876543) (Reporting)</title>\n" +
" <updated>2026-07-21T14:25:00-04:00</updated>\n" +
" <link href=\"https://www.sec.gov/Archives/edgar/data/9876543/y.htm\"/>\n" +
" </entry>\n" +
"</feed>")
got := parseEdgarAtom(body)
if len(got) != 2 {
t.Fatalf("want 2 entries, got %d", len(got))
t.Fatalf("want 2 entries, got %d (ISO-8859-1 charset must decode)", len(got))
}
if got[1].Title != "4 - Beaumonté Corp (0009876543) (Reporting)" {
t.Errorf("Latin-1 byte not decoded to é: entry[1] title = %q", got[1].Title)
}
if got[0].Title != "4 - ACME CORP (0001234567) (Issuer)" {
t.Errorf("entry[0] title = %q", got[0].Title)
@@ -63,7 +69,7 @@ func TestParseEdgarAtomWellFormed(t *testing.T) {
t.Errorf("entry[1] updated = %q", got[1].Updated)
}
name, role := splitFormTitle(got[1].Title)
if name != "Smith John A" || role != "Reporting" {
if name != "Beaumonté Corp" || role != "Reporting" {
t.Errorf("split entry[1] = (%q,%q)", name, role)
}
}