From 055a303c50e25fe0cebdc1bbad6cacc1274c9d9e Mon Sep 17 00:00:00 2001 From: Eduardo Cuducos <4732915+cuducos@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:18:17 -0400 Subject: [PATCH] Adds support for alphanumeric CNPJ (July/2026) --- db/postgres.go | 20 +++++++++----------- db/postgres/create.sql | 4 ++-- db/postgres_test.go | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- transform/transform.go | 2 +- transform/venues.go | 9 ++------- 7 files changed, 20 insertions(+), 27 deletions(-) diff --git a/db/postgres.go b/db/postgres.go index d4a6f04..65cec3a 100644 --- a/db/postgres.go +++ b/db/postgres.go @@ -7,7 +7,6 @@ import ( "fmt" "log" "path/filepath" - "strconv" "strings" "text/template" @@ -99,12 +98,16 @@ func (p *PostgreSQL) DropTable() error { // CreateCompanies performs a copy to create a batch of companies in the // database. It expects an array and each item should be another array with only // two items: the ID and the JSON field values. -func (p *PostgreSQL) CreateCompanies(batch [][]any) error { +func (p *PostgreSQL) CreateCompanies(batch [][]string) error { + b := make([][]any, len(batch)) + for i, r := range batch { + b[i] = []any{r[0], r[1]} + } _, err := p.pool.CopyFrom( context.Background(), pgx.Identifier{p.CompanyTableName}, []string{idFieldName, jsonFieldName}, - pgx.CopyFromRows(batch), + pgx.CopyFromRows(b), ) if err != nil { return fmt.Errorf("error while importing data to postgres: %w", err) @@ -114,11 +117,6 @@ func (p *PostgreSQL) CreateCompanies(batch [][]any) error { // GetCompany returns the JSON of a company based on a CNPJ number. func (p *PostgreSQL) GetCompany(id string) (string, error) { - n, err := strconv.ParseInt(id, 10, 0) - if err != nil { - return "", fmt.Errorf("error converting cnpj %s to integer: %w", id, err) - } - ctx := context.Background() if p.newRelic != nil { txn := p.newRelic.StartTransaction("GetCompany") @@ -126,13 +124,13 @@ func (p *PostgreSQL) GetCompany(id string) (string, error) { defer txn.End() } - rows, err := p.pool.Query(ctx, p.sql["get"], n) + rows, err := p.pool.Query(ctx, p.sql["get"], id) if err != nil { - return "", fmt.Errorf("error looking for cnpj %d: %w", n, err) + return "", fmt.Errorf("error looking for cnpj %s: %w", id, err) } j, err := pgx.CollectOneRow(rows, pgx.RowTo[string]) if err != nil { - return "", fmt.Errorf("error reading cnpj %d: %w", n, err) + return "", fmt.Errorf("error reading cnpj %s: %w", id, err) } return j, nil } diff --git a/db/postgres/create.sql b/db/postgres/create.sql index c034d2d..7648738 100644 --- a/db/postgres/create.sql +++ b/db/postgres/create.sql @@ -1,9 +1,9 @@ CREATE TABLE IF NOT EXISTS {{ .CompanyTableFullName }} ( tmp_pk SERIAL PRIMARY KEY, - {{ .IDFieldName }} bigint NOT NULL, + {{ .IDFieldName }} char(14) NOT NULL, {{ .JSONFieldName }} jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS {{ .MetaTableFullName }} ( - {{ .KeyFieldName }} char(16) NOT NULL PRIMARY KEY, + {{ .KeyFieldName }} char(16) NOT NULL PRIMARY KEY, {{ .ValueFieldName }} text NOT NULL ) diff --git a/db/postgres_test.go b/db/postgres_test.go index aee1a63..2506afc 100644 --- a/db/postgres_test.go +++ b/db/postgres_test.go @@ -6,7 +6,7 @@ import ( ) func TestPostgresDB(t *testing.T) { - id := 33683111000280 + id := "33683111000280" json := `{"qsa": [{"name": 42}, {"name": "forty-two"}], "answer": 42}` u := os.Getenv("TEST_DATABASE_URL") @@ -35,10 +35,10 @@ func TestPostgresDB(t *testing.T) { if err := pg.PreLoad(); err != nil { t.Errorf("expected no error pre load, got %s", err) } - if err := pg.CreateCompanies([][]any{{id, json}}); err != nil { + if err := pg.CreateCompanies([][]string{{id, json}}); err != nil { t.Errorf("expected no error saving a company, got %s", err) } - if err := pg.CreateCompanies([][]any{{id, json}}); err != nil { + if err := pg.CreateCompanies([][]string{{id, json}}); err != nil { t.Errorf("expected no error saving a duplicated company, got %s", err) } if err := pg.PostLoad(); err != nil { diff --git a/go.mod b/go.mod index daadcc6..09512cb 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/avast/retry-go/v4 v4.6.0 github.com/aws/aws-sdk-go v1.55.5 github.com/cuducos/chunk v1.1.3 - github.com/cuducos/go-cnpj v0.1.1 + github.com/cuducos/go-cnpj v0.1.2 github.com/dgraph-io/badger/v4 v4.3.1 github.com/jackc/pgx/v5 v5.7.1 github.com/newrelic/go-agent/v3 v3.35.0 diff --git a/go.sum b/go.sum index 41fae0f..7f062ad 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cuducos/chunk v1.1.3 h1:DoB2DF2Kj6xLPcDFe+VBBIMv3JfNDGMBbUKgrwmJglY= github.com/cuducos/chunk v1.1.3/go.mod h1:MieiCPvKtGwJWcSoHb4HIzC8VtQ2LeH0xzt7d6rYQz0= -github.com/cuducos/go-cnpj v0.1.1 h1:ahG8SmCa7cBMv2vuR+5GU5Se0RW2RwRwIbfFHXH1/P4= -github.com/cuducos/go-cnpj v0.1.1/go.mod h1:Oe0M530eHl9eXExbNWriqTD9kMv618ScjlGtEey/f4Y= +github.com/cuducos/go-cnpj v0.1.2 h1:EKfO9AJPjxBh/Pc8S+SMSeXvQ7kLwhDoF6te4md58OA= +github.com/cuducos/go-cnpj v0.1.2/go.mod h1:Oe0M530eHl9eXExbNWriqTD9kMv618ScjlGtEey/f4Y= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/transform/transform.go b/transform/transform.go index 9f9ccfa..edfbd54 100644 --- a/transform/transform.go +++ b/transform/transform.go @@ -25,7 +25,7 @@ const ( type database interface { PreLoad() error - CreateCompanies([][]any) error + CreateCompanies([][]string) error PostLoad() error MetaSave(string, string) error } diff --git a/transform/venues.go b/transform/venues.go index 9f33c58..f33eb4f 100644 --- a/transform/venues.go +++ b/transform/venues.go @@ -3,7 +3,6 @@ package transform import ( "fmt" "io" - "strconv" "sync" "sync/atomic" @@ -15,17 +14,13 @@ func saveBatch(db database, b []company) (int, error) { if len(b) == 0 { return 0, nil } - s := make([][]any, len(b)) + s := make([][]string, len(b)) for i, c := range b { j, err := c.JSON() if err != nil { return 0, fmt.Errorf("error getting company %s as json: %w", cnpj.Mask(c.CNPJ), err) } - n, err := strconv.Atoi(c.CNPJ) - if err != nil { - return 0, fmt.Errorf("could not convert cnpj %s to int: %w", c.CNPJ, err) - } - s[i] = []any{n, j} + s[i] = []string{c.CNPJ, j} } if err := db.CreateCompanies(s); err != nil { return 0, fmt.Errorf("error saving companies: %w", err)