diff --git a/.coverage_tests.sh b/.coverage_tests.sh new file mode 100755 index 0000000..34dbbfb --- /dev/null +++ b/.coverage_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +echo "" > coverage.txt + +for d in $(go list ./... | grep -v vendor); do + go test -race -coverprofile=profile.out -covermode=atomic $d + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..66a413d --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.out +*.log +.DS_Store diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a2df2f3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +language: go + +go: + - 1.10.x + - 1.11.x + +before_install: + - go get -t -v ./... + +script: + - go test -race -v ./... + - ./.coverage_tests.sh + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3a96138 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +## timescaledb-tune + +`timescaledb-tune` is a program for tuning a +[TimescaleDB](//github.com/timescale/timescaledb) database to perform +its best based on the host's resources such as memory and number of CPUs. +It parses the existing `postgresql.conf` file to ensure that the TimescaleDB +extension is appropriately installed and provides recommendations for memory, +parallelism, WAL, and other settings. + +### Getting started +You need the Go runtime (1.10+) installed, then simply `go get` this repo: +```bash +$ go get github.com/timescale/timescaledb-parallel-copy +``` + +### Using timescaledb-tune +By default, `timescaledb-tune` attempts to locate your `postgresql.conf` file +for parsing by using heuristics based on the operating system, so the simplest +invocation would be: +```bash +$ timescaledb-tune +``` + +You'll then be given a series of prompts that require minimal user input to +make sure your config file is up to date: +```bash +Using postgresql.conf at this path: +/usr/local/var/postgres/postgresql.conf + +Is this the correct path? [(y)es/(n)o]: y +shared_preload_libraries needs to be updated +Current: +#shared_preload_libraries = 'timescaledb' # (change requires restart) +Recommended: +shared_preload_libraries = 'timescaledb' # (change requires restart) +Is this okay? [(y)es/(n)o]: y +success: shared_preload_libraries will be updated + +Tune memory/parallelism/WAL and other settings?[(y)es/(n)o]: y +Recommendations based on 8.00 GB of available memory and 4 CPUs + +Memory settings recommendations +Current: +shared_buffers = 128MB # min 128kB +#effective_cache_size = 4GB +#maintenance_work_mem = 64MB # min 1MB +#work_mem = 4MB # min 64kB +Recommended: +shared_buffers = 2GB # min 128kB +effective_cache_size = 6GB +maintenance_work_mem = 1GB # min 1MB +work_mem = 26214kB # min 64kB +Is this okay? [(y)es/(s)kip/(q)uit]: +``` + +If you have moved the configuration file to a different location, or +auto-detection fails (file an issue please!), you can provide the location with +the `--conf-path` flag: +```bash +$ timescaledb-tune --conf-path=/path/to/postgresql.conf +``` + +At the end, your `postgresql.conf` will be overwritten with the changes that you +accepted from the prompts. + +#### Other invocations + +If you want to accept all recommendations, you can use `--yes`: +```bash +$ timescaledb-tune --yes +``` + +If you just want to see the recommendations without writing: +```bash +$ timescaledb-tune --dry-run +``` + +If there are too many prompts: +```bash +$ timescaledb-tune --quiet +``` + +And if you want to skip all prompts and get quiet output: +```bash +$ timescaledb-tune --quiet --yes +``` + +### Contributing +We welcome contributions to this utility, which like TimescaleDB is released under the Apache2 Open Source License. The same [Contributors Agreement](//github.com/timescale/timescaledb/blob/master/CONTRIBUTING.md) applies; please sign the [Contributor License Agreement](https://cla-assistant.io/timescale/timescaledb-tune) (CLA) if you're a new contributor. diff --git a/checker.go b/checker.go new file mode 100644 index 0000000..c9a5686 --- /dev/null +++ b/checker.go @@ -0,0 +1,66 @@ +package main + +import ( + "fmt" +) + +var ( + errSkip = fmt.Errorf("skip err") +) + +func isYes(s string) bool { + return s == "y" || s == "yes" +} + +func isNo(s string) bool { + return s == "n" || s == "no" +} + +func isSkip(s string) bool { + return s == "s" || s == "skip" +} + +func isQuit(s string) bool { + return s == "q" || s == "quit" +} + +type promptChecker interface { + Check(string) (bool, error) +} + +type yesNoChecker struct { + errMsg string + args []interface{} +} + +func newYesNoChecker(errMsg string, args ...interface{}) *yesNoChecker { + return &yesNoChecker{errMsg, args} +} + +func (c *yesNoChecker) Check(r string) (bool, error) { + if isNo(r) { + return false, fmt.Errorf(c.errMsg, c.args...) + } else if isYes(r) { + return true, nil + } + return false, nil +} + +type skipChecker struct { + err error +} + +func newSkipChecker(errMsg string, args ...interface{}) *skipChecker { + return &skipChecker{fmt.Errorf(errMsg, args...)} +} + +func (c *skipChecker) Check(r string) (bool, error) { + if isQuit(r) || isNo(r) { + return false, c.err + } else if isYes(r) { + return true, nil + } else if isSkip(r) { + return true, errSkip + } + return false, nil +} diff --git a/checker_test.go b/checker_test.go new file mode 100644 index 0000000..5fa11e9 --- /dev/null +++ b/checker_test.go @@ -0,0 +1,164 @@ +package main + +import "testing" + +type isCases struct { + s string + want bool +} + +func testIsFunc(t *testing.T, cases []isCases, fn func(string) bool) { + for _, c := range cases { + if got := fn(c.s); got != c.want { + t.Errorf("'%s' returned wrong answer: got %v want %v", c.s, got, c.want) + } + } +} + +func TestIsYes(t *testing.T) { + cases := []isCases{ + { + s: "y", + want: true, + }, + { + s: "yes", + want: true, + }, + { + s: "ye", + want: false, + }, + { + s: "n", + want: false, + }, + { + s: "no", + want: false, + }, + { + s: "", + want: false, + }, + } + + testIsFunc(t, cases, isYes) +} + +func TestIsNo(t *testing.T) { + cases := []isCases{ + { + s: "y", + want: false, + }, + { + s: "yes", + want: false, + }, + { + s: "ye", + want: false, + }, + { + s: "n", + want: true, + }, + { + s: "no", + want: true, + }, + { + s: "", + want: false, + }, + } + + testIsFunc(t, cases, isNo) +} + +func TestIsSkip(t *testing.T) { + cases := []isCases{ + { + s: "s", + want: true, + }, + { + s: "skip", + want: true, + }, + { + s: "sk", + want: false, + }, + { + s: "y", + want: false, + }, + { + s: "yes", + want: false, + }, + { + s: "ye", + want: false, + }, + { + s: "n", + want: false, + }, + { + s: "no", + want: false, + }, + { + s: "", + want: false, + }, + } + + testIsFunc(t, cases, isSkip) +} + +func TestIsQuit(t *testing.T) { + cases := []isCases{ + { + s: "q", + want: true, + }, + { + s: "quit", + want: true, + }, + { + s: "qu", + want: false, + }, + { + s: "y", + want: false, + }, + { + s: "yes", + want: false, + }, + { + s: "ye", + want: false, + }, + { + s: "n", + want: false, + }, + { + s: "no", + want: false, + }, + { + s: "", + want: false, + }, + } + + testIsFunc(t, cases, isQuit) +} diff --git a/config_file.go b/config_file.go new file mode 100644 index 0000000..04db678 --- /dev/null +++ b/config_file.go @@ -0,0 +1,61 @@ +package main + +import ( + "bufio" + "fmt" + "io" +) + +// configFileState represents that the postgresql.conf file, including all of its +// lines, the parsed result of the shared_preload_libraries line, and parse results +// for parameters we care about tuning +type configFileState struct { + lines []string // all the lines, to be updated for output + sharedLibResult *sharedLibResult // parsing result for shared lib line + tuneParseResults map[string]*tunableParseResult // mapping of each tunable param to its parsed line result +} + +// getConfigFileState returns the current state of the configuration file by +// reading it line by line and parsing those lines we particularly care about. +func getConfigFileState(r io.Reader) (*configFileState, error) { + cfs := &configFileState{ + lines: []string{}, + tuneParseResults: make(map[string]*tunableParseResult), + } + i := 0 + scanner := bufio.NewScanner(r) + for scanner.Scan() { + if scanner.Err() != nil { + return nil, fmt.Errorf("could not read postgresql.conf: %v", scanner.Err()) + } + line := scanner.Text() + temp := parseLineForSharedLibResult(line) + if temp != nil { + temp.idx = i + cfs.sharedLibResult = temp + } else { + for k, regex := range regexes { + tpr := parseWithRegex(line, regex) + if tpr != nil { + tpr.idx = i + cfs.tuneParseResults[k] = tpr + } + } + } + cfs.lines = append(cfs.lines, line) + i++ + } + return cfs, nil +} + +func (cfs *configFileState) WriteTo(w io.Writer) (int64, error) { + ret := int64(0) + for _, l := range cfs.lines { + n, err := w.Write([]byte(l + "\n")) + if err != nil { + return 0, err + } + ret += int64(n) + } + return ret, nil +} diff --git a/config_file_test.go b/config_file_test.go new file mode 100644 index 0000000..74fa2b7 --- /dev/null +++ b/config_file_test.go @@ -0,0 +1,189 @@ +package main + +import ( + "bytes" + "fmt" + "strings" + "testing" +) + +func TestGetConfigFileState(t *testing.T) { + sharedLibLine := "shared_preload_libraries = 'timescaledb' # comment" + memoryLine := "#shared_buffers = 64MB" + walLine := "min_wal_size = 0GB # weird" + cases := []struct { + desc string + lines []string + want *configFileState + }{ + { + desc: "empty file", + lines: []string{}, + want: &configFileState{ + lines: []string{}, + tuneParseResults: make(map[string]*tunableParseResult), + sharedLibResult: nil, + }, + }, + { + desc: "single irrelevant line", + lines: []string{"foo"}, + want: &configFileState{ + lines: []string{"foo"}, + tuneParseResults: make(map[string]*tunableParseResult), + sharedLibResult: nil, + }, + }, + { + desc: "shared lib line only", + lines: []string{sharedLibLine}, + want: &configFileState{ + lines: []string{sharedLibLine}, + tuneParseResults: make(map[string]*tunableParseResult), + sharedLibResult: &sharedLibResult{ + idx: 0, + commented: false, + hasTimescale: true, + commentGroup: "", + libs: "timescaledb", + }, + }, + }, + { + desc: "multi-line", + lines: []string{"foo", sharedLibLine, "bar", memoryLine, walLine, "baz"}, + want: &configFileState{ + lines: []string{"foo", sharedLibLine, "bar", memoryLine, walLine, "baz"}, + tuneParseResults: map[string]*tunableParseResult{ + sharedBuffersKey: &tunableParseResult{ + idx: 3, + commented: true, + key: sharedBuffersKey, + value: "64MB", + extra: "", + }, + minWalKey: &tunableParseResult{ + idx: 4, + commented: false, + key: minWalKey, + value: "0GB", + extra: " # weird", + }, + }, + sharedLibResult: &sharedLibResult{ + idx: 1, + commented: false, + hasTimescale: true, + commentGroup: "", + libs: "timescaledb", + }, + }, + }, + } + + for _, c := range cases { + buf := bytes.NewBufferString(strings.Join(c.lines, "\n")) + cfs, _ := getConfigFileState(buf) + if got := len(cfs.lines); got != len(c.want.lines) { + t.Errorf("%s: incorrect number of cfs lines: got %d want %d", c.desc, got, len(c.want.lines)) + } else { + for i, got := range cfs.lines { + if want := c.want.lines[i]; got != want { + t.Errorf("%s: incorrect line at %d: got\n%s\nwant\n%s", c.desc, i, got, want) + } + } + } + + if c.want.sharedLibResult != nil { + if cfs.sharedLibResult == nil { + t.Errorf("%s: unexpected nil shared lib result", c.desc) + } else { + want := fmt.Sprintf("%v", c.want.sharedLibResult) + if got := fmt.Sprintf("%v", cfs.sharedLibResult); got != want { + t.Errorf("%s: incorrect sharedLibResult: got %s want %s", c.desc, got, want) + } + } + } + + if len(c.want.tuneParseResults) > 0 { + if got := len(cfs.tuneParseResults); got != len(c.want.tuneParseResults) { + t.Errorf("%s: incorrect tuneParseResults size: got %d want %d", c.desc, got, len(c.want.tuneParseResults)) + } else { + for k, v := range c.want.tuneParseResults { + want := fmt.Sprintf("%v", v) + if got, ok := cfs.tuneParseResults[k]; fmt.Sprintf("%v", got) != want || !ok { + t.Errorf("%s: incorrect tuneParseResults for %s: got %s want %s", c.desc, k, fmt.Sprintf("%v", got), want) + } + } + } + } + } +} + +var errDefault = fmt.Errorf("erroring") + +type testWriter struct { + shouldErr bool + lines []string +} + +func (w *testWriter) Write(buf []byte) (int, error) { + if w.shouldErr { + return 0, errDefault + } + w.lines = append(w.lines, string(buf)) + return 0, nil +} + +func TestConfigFileStateWriteTo(t *testing.T) { + cases := []struct { + desc string + lines []string + shouldErr bool + }{ + { + desc: "empty", + lines: []string{}, + shouldErr: false, + }, + { + desc: "one line", + lines: []string{"foo"}, + shouldErr: false, + }, + { + desc: "many lines", + lines: []string{"foo", "bar", "baz", "quaz"}, + shouldErr: false, + }, + { + desc: "error", + lines: []string{"foo"}, + shouldErr: true, + }, + } + + for _, c := range cases { + cfs := &configFileState{lines: c.lines} + w := &testWriter{c.shouldErr, []string{}} + _, err := cfs.WriteTo(w) + if err != nil && !c.shouldErr { + t.Errorf("%s: unexpected error: %v", c.desc, err) + } else if err == nil && c.shouldErr { + t.Errorf("%s: unexpected lack of error", c.desc) + } else if c.shouldErr && err != errDefault { + t.Errorf("%s: unexpected type of error: %v", c.desc, err) + } + + if len(c.lines) > 0 && !c.shouldErr { + if got := len(w.lines); got != len(c.lines) { + t.Errorf("%s: incorrect output len: got %d want %d", c.desc, got, len(c.lines)) + } + for i, want := range c.lines { + if got := w.lines[i]; got != want+"\n" { + t.Errorf("%s: incorrect line at %d: got %s want %s", c.desc, i, got, want+"\n") + } + } + } + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..ecc11d0 --- /dev/null +++ b/main.go @@ -0,0 +1,497 @@ +// timescaledb-tune analyzes a user's postgresql.conf file to make sure it is +// ready and tuned to use TimescaleDB. It checks that the library is properly +// listed as a shared preload library and analyzes the memory settings to make +// sure they are reasonably set for the machine's resources. +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/pbnjay/memory" +) + +const ( + errConfigNotFoundFmt = "could not find postgresql.conf at any of these locations:\n%v" + + extName = "timescaledb" + + currentLabel = "Current:" + recommendLabel = "Recommended:" + + promptOkay = "Is this okay? " + promptYesNo = "[(y)es/(n)o]: " + promptSkip = "[(y)es/(s)kip/(q)uit]: " + + errSharedLibNeeded = "`timescaledb` needs to be added to shared_preload_libraries in order for it to work" + successSharedLibCorrect = "shared_preload_libraries is set correctly" + successSharedLibUpdated = "shared_preload_libraries will be updated" + statementSharedLibNotFound = "Unable to find shared_preload_libraries in configuration file" + plainSharedLibLine = "shared_preload_libraries = 'timescaledb' # (change requires restart)" + + statementTunableIntro = "Recommendations based on %s of available memory and %d CPUs" + promptTune = "Tune memory/parallelism/WAL and other settings?" + tunableMemory = "memory" + tunableWAL = "WAL" + tunableParallelism = "parallelism" + tunableOther = "miscellaneous" + + fmtTunableParam = "%s = %s%s\n" + + fudgeFactor = 0.05 +) + +type flags struct { + confPath string // path to the postgresql.conf file + destPath string // path to output file + pgconfig string // path to the pg_config binary + yesAlways bool // always respond yes to prompts + quiet bool // show only the bare necessities + useColor bool // use color in output + dryRun bool // whether to actual persist changes to disk +} + +// sharedLibResult holds the results of extracting/parsing the shared_preload_libraries +// line of a postgresql.conf file. +type sharedLibResult struct { + idx int // the line index where this result was parsed + commented bool // whether the line is currently commented out (i.e., prepended by #) + hasTimescale bool // whether 'timescaledb' appears in the list of libraries + commentGroup string // the combination of # + spaces that appear before the key / value + libs string // the string value of the libraries currently set in the config file +} + +// ioHandler manages the reading and writing of the application +type ioHandler struct { + p printer // handles output + br *bufio.Reader // handles input +} + +func (h *ioHandler) errorExit(err error) { + h.exit(1, err.Error()) +} + +func (h *ioHandler) exit(errCode int, format string, args ...interface{}) { + h.p.Error("exit", format, args...) + os.Exit(errCode) +} + +// Flag vars +var ( + f flags + sharedRegex = regexp.MustCompile("(#+?\\s*)?shared_preload_libraries = '(.*?)'.*") + errNeedEdit = fmt.Errorf("need to edit") + + printFn = fmt.Printf + pgVersions = []string{"10", "9.6"} +) + +// Parse args +func init() { + flag.StringVar(&f.confPath, "conf-path", "", "Path to postgresql.conf. If blank, heuristics will be used to find it") + flag.StringVar(&f.pgconfig, "pgconfig", "pg_config", "Path to pg_config binary") + flag.StringVar(&f.destPath, "out-path", "", "Path to write the new configuration file. If blank, will use the same file that is read from") + flag.BoolVar(&f.yesAlways, "yes", false, "Answer 'yes' to every prompt") + flag.BoolVar(&f.quiet, "quiet", false, "Show only the total recommendations at the end") + flag.BoolVar(&f.useColor, "color", true, "Use color in output (works best on dark terminals)") + flag.BoolVar(&f.dryRun, "dry-run", false, "Whether to just show the changes without overwriting the configuration file") + flag.Parse() +} + +func fileExists(name string) bool { + // for our purposes, any error is a problem, so assume it does not exist + if _, err := os.Stat(name); err != nil { + return false + } + return true +} + +func getConfigFilePath(os string) (string, error) { + tried := []string{} + try := func(format string, args ...interface{}) string { + fileName := fmt.Sprintf(format, args...) + tried = append(tried, fileName) + if fileExists(fileName) { + return fileName + } + return "" + } + switch { + case os == "darwin": + fileName := try("/usr/local/var/postgres/postgresql.conf") + if fileName != "" { + return fileName, nil + } + case os == "linux": + for _, v := range pgVersions { + fileName := try("/etc/postgresql/%s/main/postgresql.conf", v) + if fileName != "" { + return fileName, nil + } + } + for _, v := range pgVersions { + fileName := try("/var/lib/pgsql/%s/data/postgresql.conf", v) + if fileName != "" { + return fileName, nil + } + } + } + return "", fmt.Errorf(errConfigNotFoundFmt, strings.Join(tried, "\n")) +} + +// promptUntilValidInput continually prompts the user via handler's output to +// answer a question provided in prompt until an acceptable answer is given. +func promptUntilValidInput(handler *ioHandler, prompt string, checker promptChecker) error { + if f.yesAlways { + return nil + } + for { + handler.p.Prompt(prompt) + resp, err := handler.br.ReadString('\n') + if err != nil { + return fmt.Errorf("could not parse response: %v", err) + } + r := strings.ToLower(strings.TrimSpace(resp)) + ok, err := checker.Check(r) + if ok || err != nil { + return err + } + } +} + +func parseLineForSharedLibResult(line string) *sharedLibResult { + res := sharedRegex.FindStringSubmatch(line) + if len(res) > 0 { + return &sharedLibResult{ + commented: len(res[1]) > 0, + hasTimescale: strings.Contains(res[2], extName), + commentGroup: res[1], + libs: res[2], + } + } + return nil +} + +func updateSharedLibLine(line string, parseResult *sharedLibResult) string { + res := line + if parseResult.commented { + res = strings.Replace(res, parseResult.commentGroup, "", 1) + } + + if parseResult.hasTimescale { + return res + } + newLibsVal := "= '" + if len(parseResult.libs) > 0 { + newLibsVal += parseResult.libs + "," + } + newLibsVal += extName + "'" + replaceVal := "= '" + parseResult.libs + "'" + res = strings.Replace(res, replaceVal, newLibsVal, 1) + + return res +} + +func processNoSharedLibLine(handler *ioHandler, cfs *configFileState) error { + handler.p.Statement(statementSharedLibNotFound) + checker := newYesNoChecker(errSharedLibNeeded) + err := promptUntilValidInput(handler, "Append to end? "+promptYesNo, checker) + if err != nil { + return err + } + + cfs.lines = append(cfs.lines, plainSharedLibLine) + handler.p.Success("appending shared_preload_libraries = 'timescaledb' to end of configuration file") + + return nil +} + +func processSharedLibLine(handler *ioHandler, cfs *configFileState) error { + if cfs.sharedLibResult == nil { + return processNoSharedLibLine(handler, cfs) + } + + sharedIdx := cfs.sharedLibResult.idx + newLine := updateSharedLibLine(cfs.lines[sharedIdx], cfs.sharedLibResult) + if newLine == cfs.lines[sharedIdx] { + handler.p.Success(successSharedLibCorrect) + } else { + handler.p.Statement("shared_preload_libraries needs to be updated") + handler.p.Statement(currentLabel) + printFn(cfs.lines[sharedIdx] + "\n") + handler.p.Statement(recommendLabel) + printFn(newLine + "\n") + checker := newYesNoChecker(errSharedLibNeeded) + err := promptUntilValidInput(handler, promptOkay+promptYesNo, checker) + if err != nil { + return err + } + cfs.lines[sharedIdx] = newLine + handler.p.Success(successSharedLibUpdated) + } + return nil +} + +func checkIfShouldShowSetting(keys []string, parseResults map[string]*tunableParseResult, recommender recommender) (map[string]bool, error) { + show := make(map[string]bool) + for _, k := range keys { + r := parseResults[k] + + // if the setting was not found on pass through, should show our rec + if r == nil { + show[k] = true + continue + } + + // parse the value already there; if unparseable, should show our rec + curr, err := parsers[k](r.value) + if err != nil { + show[k] = true + continue + } + + // get and parse our recommendation; fail if for we can't + rec := recommender.Recommend(k) + target, err := parsers[k](rec) + if err != nil { + return nil, fmt.Errorf("unexpected parsing problem: %v", err) + } + + // only show if our recommendation is significantly different, or config is commented + if !isCloseEnough(curr, target, fudgeFactor) || r.commented { + show[k] = true + } + } + return show, nil +} + +type tuneSettings struct { + label string + rec recommender + keys []string +} + +func processSettingsGroup(handler *ioHandler, cfs *configFileState, ts *tuneSettings, quiet bool) error { + if !quiet { + printFn("\n") + handler.p.Statement(fmt.Sprintf("%s%s settings recommendations", strings.ToUpper(ts.label[:1]), ts.label[1:])) + } + + // Get a map of only the settings that are missing, commented out, or not "close enough" to our recommendation. + show, err := checkIfShouldShowSetting(ts.keys, cfs.tuneParseResults, ts.rec) + if err != nil { + return err + } + + // Settings that need to be changed exist... + if len(show) > 0 { + // Decorator for a function fn, where only the lines that need to be updated + // are processed + doWithVisibile := func(fn func(r *tunableParseResult)) { + for _, k := range ts.keys { + if _, ok := show[k]; !ok { + continue + } + r, ok := cfs.tuneParseResults[k] + if !ok { + r = &tunableParseResult{idx: -1, missing: true, key: k} + } + fn(r) + } + } + + // Display extra helpful info in non-quiet mode + if !quiet { + // Display current settings, but only those with new recommendations + handler.p.Statement(currentLabel) + doWithVisibile(func(r *tunableParseResult) { + if r.idx == -1 { + handler.p.Error("missing", r.key) + return + } + printFn(cfs.lines[r.idx] + "\n") + }) + + // Now display recommendations, but only those with new recommendations + handler.p.Statement(recommendLabel) + } + // Recommendations are always displayed, but the label above may not be + doWithVisibile(func(r *tunableParseResult) { + printFn(fmtTunableParam, r.key, ts.rec.Recommend(r.key), r.extra) + }) + + // Prompt the user for input (only in non-quiet mode) + if !quiet { + checker := newSkipChecker(ts.label + " settings still need to be tuned, please re-run or do so manually") + err := promptUntilValidInput(handler, promptOkay+promptSkip, checker) + if err == errSkip { + handler.p.Error("warning", ts.label+" settings left alone, but still need tuning") + return nil + } else if err != nil { + return err + } + handler.p.Success(ts.label + " settings will be updated") + } + + // If we reach here, it means the user accepted our recommendations, so update the lines + doWithVisibile(func(r *tunableParseResult) { + newLine := fmt.Sprintf(fmtTunableParam, r.key, ts.rec.Recommend(r.key), r.extra) + if r.idx == -1 { + cfs.lines = append(cfs.lines, newLine) + } else { + cfs.lines[r.idx] = newLine + } + }) + } else if !quiet { // nothing to tune + handler.p.Success(ts.label + " settings are already tuned") + } + + return nil +} + +func processTunables(handler *ioHandler, cfs *configFileState, totalMemory uint64, cpus int, quiet bool) { + tune := func(label string, r recommender, keys []string) { + ts := tuneSettings{label, r, keys} + err := processSettingsGroup(handler, cfs, &ts, quiet) + if err != nil { + handler.errorExit(err) + } + } + if !quiet { + handler.p.Statement(statementTunableIntro, bytesFormat(totalMemory), cpus) + } + + mr := &memoryRecommender{totalMemory, cpus} + tune(tunableMemory, mr, memoryKeys) + + pr := ¶llelRecommender{cpus} + tune(tunableParallelism, pr, parallelKeys) + + wr := &walRecommender{totalMemory} + tune(tunableWAL, wr, walKeys) + + mir := &miscRecommender{} + tune(tunableOther, mir, otherKeys) +} + +func processQuiet(handler *ioHandler, cfs *configFileState, totalMemory uint64, cpus int) error { + handler.p.Statement(statementTunableIntro, bytesFormat(totalMemory), cpus) + if cfs.sharedLibResult == nil { + printFn(plainSharedLibLine + "\n") + cfs.lines = append(cfs.lines, plainSharedLibLine) + cfs.sharedLibResult = parseLineForSharedLibResult(plainSharedLibLine) + cfs.sharedLibResult.idx = len(cfs.lines) - 1 + } else { + sharedIdx := cfs.sharedLibResult.idx + newLine := updateSharedLibLine(cfs.lines[sharedIdx], cfs.sharedLibResult) + if newLine != cfs.lines[sharedIdx] { + printFn(newLine + "\n") + cfs.lines[sharedIdx] = newLine + } + } + + processTunables(handler, cfs, totalMemory, cpus, true /* quiet */) + checker := newYesNoChecker("not using these settings could lead to suboptimal performance") + err := promptUntilValidInput(handler, "Use these recommendations? "+promptYesNo, checker) + if err != nil { + return err + } + + return nil +} + +func main() { + var err error + // setup IO + var p printer + if f.useColor { + p = &colorPrinter{} + } else { + p = &noColorPrinter{} + } + handler := &ioHandler{p: p} + + // attempt to find the config file and open it for reading + fileName := f.confPath + if len(fileName) == 0 { + fileName, err = getConfigFilePath(runtime.GOOS) + if err != nil { + handler.errorExit(err) + } + } + + file, err := os.Open(fileName) + if err != nil { + handler.errorExit(fmt.Errorf("could not open config file for reading: %v", err)) + } + defer file.Close() + + br := bufio.NewReader(os.Stdin) + handler.br = br + + handler.p.Statement("Using postgresql.conf at this path:") + printFn(fileName + "\n\n") + if len(f.confPath) == 0 { + checker := newYesNoChecker("please pass in the correct path to postgresql.conf using the --conf-path flag") + err = promptUntilValidInput(handler, "Is this the correct path? "+promptYesNo, checker) + if err != nil { + handler.exit(0, err.Error()) + } + } + + // write backup + + cfs, err := getConfigFileState(file) + if err != nil { + handler.errorExit(err) + } + + totalMemory := memory.TotalMemory() + cpus := runtime.NumCPU() + + if f.quiet { + err = processQuiet(handler, cfs, totalMemory, cpus) + if err != nil { + handler.errorExit(err) + } + } else { + err = processSharedLibLine(handler, cfs) + if err != nil { + handler.errorExit(err) + } + + printFn("\n") + err = promptUntilValidInput(handler, promptTune+promptYesNo, newYesNoChecker("")) + if err == nil { + processTunables(handler, cfs, totalMemory, cpus, false /* quiet */) + } + } + + if !f.dryRun { + outPath := f.destPath + if len(outPath) == 0 { + outPath, err = filepath.Abs(fileName) + if err != nil { + handler.exit(1, "could not open %s for writing: %v", fileName, err) + } + } + + handler.p.Statement("Saving changes to: " + outPath) + f, err := os.Create(outPath) + if err != nil { + handler.exit(1, "could not open %s for writing: %v", outPath, err) + } + _, err = cfs.WriteTo(f) + if err != nil { + handler.errorExit(err) + } + } else { + handler.p.Statement("Success, but not writing due to --dry-run flag") + } +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..12d237f --- /dev/null +++ b/main_test.go @@ -0,0 +1,1074 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "strings" + "testing" +) + +type limitChecker struct { + limit uint64 + calls uint64 + shouldErr bool + checks []string +} + +func (c *limitChecker) Check(r string) (bool, error) { + c.calls++ + c.checks = append(c.checks, r) + if c.calls >= c.limit { + if c.shouldErr { + return false, fmt.Errorf("errored") + } + return true, nil + } + return false, nil +} + +func TestPromptUntilValidInput(t *testing.T) { + cases := []struct { + desc string + limit uint64 + shouldErr bool + }{ + { + desc: "always returns true", + limit: 1, + shouldErr: false, + }, + { + desc: "always errors", + limit: 1, + shouldErr: true, + }, + { + desc: "skip once, then success", + limit: 2, + shouldErr: false, + }, + { + desc: "skip once, then error", + limit: 2, + shouldErr: true, + }, + { + desc: "skip twice", + limit: 3, + shouldErr: false, + }, + { + desc: "check all are lower", + limit: 5, + shouldErr: false, + }, + } + + testString := "foo\nFoo\nFOO\nfOo\nfOO\n\n" + + for _, c := range cases { + buf := bytes.NewBuffer([]byte(testString)) + br := bufio.NewReader(buf) + handler := &ioHandler{ + p: &testPrinter{}, + br: br, + } + checker := &limitChecker{limit: c.limit, shouldErr: c.shouldErr} + err := promptUntilValidInput(handler, "test prompt", checker) + if err != nil && !c.shouldErr { + t.Errorf("%s: unexpected error: %v", c.desc, err) + } else if err == nil && c.shouldErr { + t.Errorf("%s: unexpected lack of error", c.desc) + } + + if got := handler.p.(*testPrinter).promptCalls; got != c.limit { + t.Errorf("%s: incorrect number of prompts: got %d want %d", c.desc, got, c.limit) + } + + if got := len(checker.checks); got != int(c.limit) { + t.Errorf("%s: incorrect number of checks: got %d want %d", c.desc, got, c.limit) + } + + for i, check := range checker.checks { + if check != strings.ToLower(check) { + t.Errorf("%s: check was not lowercase: %s (idx %d)", c.desc, check, i) + } + } + } +} + +func TestParseLineForSharedLibResult(t *testing.T) { + cases := []struct { + desc string + input string + want *sharedLibResult + }{ + { + desc: "initial config value", + input: "#shared_preload_libraries = '' # (change requires restart)", + want: &sharedLibResult{ + commented: true, + hasTimescale: false, + libs: "", + }, + }, + { + desc: "extra commented out", + input: "###shared_preload_libraries = '' # (change requires restart)", + want: &sharedLibResult{ + commented: true, + hasTimescale: false, + libs: "", + }, + }, + { + desc: "commented with space after", + input: "# shared_preload_libraries = '' # (change requires restart)", + want: &sharedLibResult{ + commented: true, + hasTimescale: false, + libs: "", + }, + }, + { + desc: "extra commented with space after", + input: "## shared_preload_libraries = '' # (change requires restart)", + want: &sharedLibResult{ + commented: true, + hasTimescale: false, + libs: "", + }, + }, + { + desc: "initial config value, uncommented", + input: "shared_preload_libraries = '' # (change requires restart)", + want: &sharedLibResult{ + commented: false, + hasTimescale: false, + libs: "", + }, + }, + { + desc: "initial config value, uncommented with leading space", + input: " shared_preload_libraries = '' # (change requires restart)", + want: &sharedLibResult{ + commented: false, + hasTimescale: false, + libs: "", + }, + }, + { + desc: "timescaledb already there but commented", + input: "#shared_preload_libraries = 'timescaledb' # (change requires restart)", + want: &sharedLibResult{ + commented: true, + hasTimescale: true, + libs: "timescaledb", + }, + }, + { + desc: "other libraries besides timescaledb, commented", + input: "#shared_preload_libraries = 'pg_stats' # (change requires restart) ", + want: &sharedLibResult{ + commented: true, + hasTimescale: false, + libs: "pg_stats", + }, + }, + { + desc: "no string after the quotes", + input: "shared_preload_libraries = 'pg_stats,timescaledb'", + want: &sharedLibResult{ + commented: false, + hasTimescale: true, + libs: "pg_stats,timescaledb", + }, + }, + { + desc: "don't be greedy with things between single quotes", + input: "#shared_preload_libraries = 'timescaledb' # comment with single quote ' test", + want: &sharedLibResult{ + commented: true, + hasTimescale: true, + libs: "timescaledb", + }, + }, + { + desc: "not shared preload line", + input: "data_dir = '/path/to/data'", + want: nil, + }, + } + for _, c := range cases { + res := parseLineForSharedLibResult(c.input) + if res == nil && c.want != nil { + t.Errorf("%s: result was unexpectedly nil: want %v", c.desc, c.want) + } else if res != nil && c.want == nil { + t.Errorf("%s: result was unexpectedly non-nil: got %v", c.desc, res) + } else if c.want != nil { + if got := res.commented; got != c.want.commented { + t.Errorf("%s: incorrect commented: got %v want %v", c.desc, got, c.want.commented) + } + if got := res.hasTimescale; got != c.want.hasTimescale { + t.Errorf("%s: incorrect hasTimescale: got %v want %v", c.desc, got, c.want.hasTimescale) + } + if got := res.libs; got != c.want.libs { + t.Errorf("%s: incorrect libs: got %s want %s", c.desc, got, c.want.libs) + } + } + } +} + +func TestUpdateSharedLibLine(t *testing.T) { + confKey := "shared_preload_libraries = " + simpleOkayCase := confKey + "'" + extName + "'" + simpleOkayCaseExtra := simpleOkayCase + " # (change requires restart)" + cases := []struct { + desc string + original string + want string + }{ + { + desc: "original = ok", + original: simpleOkayCase, + want: simpleOkayCase, + }, + { + desc: "original = ok w/ ending comments", + original: simpleOkayCaseExtra, + want: simpleOkayCaseExtra, + }, + { + desc: "original = ok w/ prepended spaces", + original: " " + simpleOkayCase, + want: " " + simpleOkayCase, + }, + { + desc: "just need to uncomment", + original: "#" + simpleOkayCase, + want: simpleOkayCase, + }, + { + desc: "just need to uncomment w/ ending comments", + original: "#" + simpleOkayCaseExtra, + want: simpleOkayCaseExtra, + }, + { + desc: "just need to uncomment multiple times", + original: "###" + simpleOkayCase, + want: simpleOkayCase, + }, + { + desc: "uncomment + spaces", + original: "### " + simpleOkayCase, + want: simpleOkayCase, + }, + { + desc: "needs to be added, empty list", + original: confKey + "''", + want: simpleOkayCase, + }, + { + desc: "needs to be added, empty list, commented out", + original: "#" + confKey + "''", + want: simpleOkayCase, + }, + { + desc: "needs to be added, empty list, trailing comment", + original: confKey + "'' # (change requires restart)", + want: simpleOkayCaseExtra, + }, + { + desc: "needs to be added, one item", + original: confKey + "'pg_stats'", + want: confKey + "'pg_stats," + extName + "'", + }, + { + desc: "needs to be added, t item, commented out", + original: "#" + confKey + "'pg_stats,ext2'", + want: confKey + "'pg_stats,ext2," + extName + "'", + }, + { + desc: "needs to be added, two items", + original: confKey + "'pg_stats'", + want: confKey + "'pg_stats," + extName + "'", + }, + { + desc: "needs to be added, two items, commented out", + original: "#" + confKey + "'pg_stats,ext2'", + want: confKey + "'pg_stats,ext2," + extName + "'", + }, + { + desc: "in list with others", + original: confKey + "'timescaledb,pg_stats'", + want: confKey + "'timescaledb,pg_stats'", + }, + { + desc: "in list with others, commented out", + original: "#" + confKey + "'timescaledb,pg_stats'", + want: confKey + "'timescaledb,pg_stats'", + }, + } + + for _, c := range cases { + res := parseLineForSharedLibResult(c.original) + if res == nil { + t.Errorf("%s: parsing gave unexpected nil", c.desc) + } + got := updateSharedLibLine(c.original, res) + if got != c.want { + t.Errorf("%s: incorrect result: got\n%s\nwant\n%s", c.desc, got, c.want) + } + } +} + +func TestProcessNoSharedLibLine(t *testing.T) { + cases := []struct { + desc string + input string + shouldErr bool + prompts uint64 + }{ + { + desc: "success on first prompt (y)", + input: "y\n", + shouldErr: false, + prompts: 1, + }, + { + desc: "success on first prompt (yes)", + input: "yes\n", + shouldErr: false, + prompts: 1, + }, + { + desc: "success on later try", + input: " \nYES\n", + shouldErr: false, + prompts: 2, + }, + { + desc: "error on first prompt (n)", + input: "n\n\n", + shouldErr: true, + prompts: 1, + }, + { + desc: "error on first prompt (no)", + input: "no\n", + shouldErr: true, + prompts: 1, + }, + { + desc: "error on later prompt (n)", + input: "x\nx\nNO\n", + shouldErr: true, + prompts: 3, + }, + { + desc: "error closed stream", + input: "", + shouldErr: true, + prompts: 1, + }, + } + for _, c := range cases { + buf := bytes.NewBuffer([]byte(c.input)) + br := bufio.NewReader(buf) + handler := &ioHandler{ + p: &testPrinter{}, + br: br, + } + cfs := &configFileState{lines: []string{}} + err := processNoSharedLibLine(handler, cfs) + if got := handler.p.(*testPrinter).statementCalls; got != 1 { + t.Errorf("%s: incorrect number of statements: got %d want %d", c.desc, got, 1) + } + + if err != nil && !c.shouldErr { + t.Errorf("%s: unexpected error: %v", c.desc, err) + } else if err == nil && c.shouldErr { + t.Errorf("%s: unexpected lack of error", c.desc) + } + + if got := handler.p.(*testPrinter).promptCalls; got != c.prompts { + t.Errorf("%s: incorrect number of prompts: got %d want %d", c.desc, got, c.prompts) + } + if err == nil { + if got := handler.p.(*testPrinter).successCalls; got != 1 { + t.Errorf("%s: incorrect number of successes: got %d want %d", c.desc, got, 1) + } + } + } +} + +func TestProcessSharedLibLine(t *testing.T) { + okLine := "shared_preload_libraries = 'timescaledb' # (need restart)" + cases := []struct { + desc string + lines []string + input string + shouldErr bool + prompts uint64 + prints []string + statements uint64 + successMsg string + }{ + { + desc: "no change", + lines: []string{okLine}, + input: "\n", + shouldErr: false, + prompts: 0, + statements: 0, + successMsg: successSharedLibCorrect, + }, + { + desc: "success on prompt", + lines: []string{"#" + okLine}, + input: "y\n", + shouldErr: false, + prompts: 1, + statements: 3, + prints: []string{"#" + okLine + "\n", okLine + "\n"}, + successMsg: successSharedLibUpdated, + }, + { + desc: "success on 2nd prompt", + lines: []string{"#" + okLine}, + input: " \ny\n", + shouldErr: false, + prompts: 2, + statements: 3, + prints: []string{"#" + okLine + "\n", okLine + "\n"}, + successMsg: successSharedLibUpdated, + }, + { + desc: "fail", + lines: []string{"#" + okLine}, + input: " \nn\n", + shouldErr: true, + prompts: 2, + statements: 3, + prints: []string{"#" + okLine + "\n", okLine + "\n"}, + successMsg: "", + }, + { + desc: "no shared lib, success", + lines: []string{""}, + input: "y\n", + shouldErr: false, + prompts: 1, + statements: 1, + successMsg: "", + }, + { + desc: "no shared lib, fail", + lines: []string{""}, + input: "n\n", + shouldErr: true, + prompts: 1, + statements: 1, + successMsg: "", + }, + } + + oldPrintFn := printFn + + for _, c := range cases { + buf := bytes.NewBufferString(c.input) + br := bufio.NewReader(buf) + handler := &ioHandler{ + p: &testPrinter{}, + br: br, + } + cfs := &configFileState{lines: c.lines} + cfs.sharedLibResult = parseLineForSharedLibResult(c.lines[0]) + + prints := []string{} + printFn = func(format string, args ...interface{}) (int, error) { + prints = append(prints, fmt.Sprintf(format, args...)) + return 0, nil + } + + err := processSharedLibLine(handler, cfs) + if err != nil && !c.shouldErr { + t.Errorf("%s: unexpected error: %v", c.desc, err) + } else if err == nil && c.shouldErr { + t.Errorf("%s: unexpected lack of err", c.desc) + } + + tp := handler.p.(*testPrinter) + if got := tp.promptCalls; got != c.prompts { + t.Errorf("%s: incorrect number of prompts: got %d want %d", c.desc, got, c.prompts) + } + if got := tp.statementCalls; got != c.statements { + t.Errorf("%s: incorrect number of statements: got %d want %d", c.desc, got, c.statements) + } + + if len(c.prints) > 0 { + for i, want := range c.prints { + if got := prints[i]; got != want { + t.Errorf("%s: incorrect print at %d: got\n%s\nwant\n%s", c.desc, i, got, want) + } + } + } + + if len(c.successMsg) > 0 { + if got := tp.successes[0]; got != c.successMsg { + t.Errorf("%s: incorrect success msg: got\n%s\nwant\n%s", c.desc, got, c.successMsg) + } + } + } + + printFn = oldPrintFn +} + +func TestCheckIfShouldShowSetting(t *testing.T) { + valSharedBuffers := "2GB" + valEffective := "6GB" + valWorkMem := "52428kB" + valMaintenance := "1GB" + okSharedBuffers := &tunableParseResult{ + idx: 0, + commented: false, + key: sharedBuffersKey, + value: valSharedBuffers, + } + okEffective := &tunableParseResult{ + idx: 1, + commented: false, + key: effectiveCacheKey, + value: valEffective, + } + okWorkMem := &tunableParseResult{ + idx: 2, + commented: false, + key: workMemKey, + value: valWorkMem, + } + okMaintenance := &tunableParseResult{ + idx: 3, + commented: false, + key: maintenanceWorkMemKey, + value: valMaintenance, + } + badWorkMem := &tunableParseResult{ + idx: 2, + commented: false, + key: workMemKey, + value: "0B", + } + cases := []struct { + desc string + parseResults map[string]*tunableParseResult + okFudge []string + highFudge []string + lowFudge []string + commented []string + want []string + errMsg string + }{ + { + desc: "show nothing", + parseResults: map[string]*tunableParseResult{ + sharedBuffersKey: okSharedBuffers, + effectiveCacheKey: okEffective, + workMemKey: okWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + want: []string{}, + }, + { + desc: "show 1, missing", + parseResults: map[string]*tunableParseResult{ + effectiveCacheKey: okEffective, + workMemKey: okWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + want: []string{sharedBuffersKey}, + }, + { + desc: "show 1, unparseable", + parseResults: map[string]*tunableParseResult{ + sharedBuffersKey: okSharedBuffers, + effectiveCacheKey: okEffective, + workMemKey: badWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + want: []string{workMemKey}, + }, + { + desc: "show 2, 1 unparseable + 1 missing", + parseResults: map[string]*tunableParseResult{ + effectiveCacheKey: okEffective, + workMemKey: badWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + want: []string{sharedBuffersKey, workMemKey}, + }, + { + desc: "show all, all commented", + parseResults: map[string]*tunableParseResult{ + sharedBuffersKey: okSharedBuffers, + effectiveCacheKey: okEffective, + workMemKey: okWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + commented: []string{sharedBuffersKey, effectiveCacheKey, workMemKey, maintenanceWorkMemKey}, + want: []string{sharedBuffersKey, effectiveCacheKey, workMemKey, maintenanceWorkMemKey}, + }, + { + desc: "show one, 1 commented", + parseResults: map[string]*tunableParseResult{ + sharedBuffersKey: okSharedBuffers, + effectiveCacheKey: okEffective, + workMemKey: okWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + commented: []string{effectiveCacheKey}, + want: []string{effectiveCacheKey}, + }, + { + desc: "show none, 1 ok fudge", + + parseResults: map[string]*tunableParseResult{ + sharedBuffersKey: okSharedBuffers, + effectiveCacheKey: okEffective, + workMemKey: okWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + okFudge: []string{}, + commented: []string{}, + want: []string{}, + }, + { + desc: "show 2, 1 high fudge, 1 low fudge", + + parseResults: map[string]*tunableParseResult{ + sharedBuffersKey: okSharedBuffers, + effectiveCacheKey: okEffective, + workMemKey: okWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + highFudge: []string{sharedBuffersKey}, + lowFudge: []string{workMemKey}, + commented: []string{}, + want: []string{sharedBuffersKey, workMemKey}, + }, + { + desc: "show 2, 1 high fudge commented too, 1 low fudge", + + parseResults: map[string]*tunableParseResult{ + sharedBuffersKey: okSharedBuffers, + effectiveCacheKey: okEffective, + workMemKey: okWorkMem, + maintenanceWorkMemKey: okMaintenance, + }, + highFudge: []string{sharedBuffersKey}, + lowFudge: []string{workMemKey}, + commented: []string{sharedBuffersKey}, + want: []string{sharedBuffersKey, workMemKey}, + }, + } + + reset := func() { + okSharedBuffers.commented = false + okSharedBuffers.value = valSharedBuffers + okEffective.commented = false + okEffective.value = valEffective + okWorkMem.commented = false + okWorkMem.value = valWorkMem + okMaintenance.commented = false + okMaintenance.value = valMaintenance + } + + for _, c := range cases { + reset() + // change those keys who are supposed to be commented out + for _, k := range c.commented { + c.parseResults[k].commented = true + } + // change values, but still within fudge factor so it shouldn't be shown + for _, k := range c.okFudge { + temp, err := parsePGStringToBytes(c.parseResults[k].value) + if err != nil { + t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + } + temp = temp + float64(temp)*(fudgeFactor-.01) + c.parseResults[k].value = bytesPGFormat(uint64(temp)) + } + // change values to higher fudge factor, so it should be shown + for _, k := range c.highFudge { + temp, err := parsePGStringToBytes(c.parseResults[k].value) + if err != nil { + t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + } + temp = temp + float64(temp)*(fudgeFactor+.01) + c.parseResults[k].value = bytesPGFormat(uint64(temp)) + } + // change values to lower fudge factor, so it should be shown + for _, k := range c.lowFudge { + temp, err := parsePGStringToBytes(c.parseResults[k].value) + if err != nil { + t.Errorf("%s: unexpected err in parsing: %v", c.desc, err) + } + temp = temp - float64(temp)*(fudgeFactor+.01) + c.parseResults[k].value = bytesPGFormat(uint64(temp)) + } + mr := &memoryRecommender{8 * gigabyte, 1} + show, err := checkIfShouldShowSetting(memoryKeys, c.parseResults, mr) + if len(c.errMsg) > 0 { + + } else if err != nil { + t.Errorf("%s: unexpected err: %v", c.desc, err) + } else { + if got := len(show); got != len(c.want) { + t.Errorf("%s: incorrect show length: got %d want %d", c.desc, got, len(c.want)) + } + for _, k := range c.want { + if _, ok := show[k]; !ok { + t.Errorf("%s: key not found: %s", c.desc, k) + } + } + } + } +} + +var ( + memSettingsCorrect = []string{ + "shared_buffers = 2GB", + "work_mem = 26214kB", + "effective_cache_size = 6GB", + "maintenance_work_mem = 1GB", + } + memSettingsCommented = []string{ + "#shared_buffers = 2GB # should be uncommented", + "work_mem = 26214kB", + "effective_cache_size = 6GB", + "maintenance_work_mem = 1GB", + } + memSettingsWrongVal = []string{ + "shared_buffers = 2GB", + "work_mem = 0kB # 0kb is wrong", + "effective_cache_size = 6GB", + "maintenance_work_mem = 1GB", + } + memSettingsMissing = []string{ + "shared_buffers = 2GB", + "work_mem = 26214kB", + // missing effective cache size + "maintenance_work_mem = 1GB", + } + memSettingsCommentWrong = []string{ + "#shared_buffers = 0GB # should be uncommented, and 2GB", + "work_mem = 26214kB", + "effective_cache_size = 6GB", + "maintenance_work_mem = 0GB # should be non-0", + } + memSettingsCommentWrongMissing = []string{ + "shared_buffers = 2GB", + // missing work_mem + "effective_cache_size = 0GB # should be non-0", + "#maintenance_work_mem = 1GB # should be uncommented", + } + memSettingsAllWrong = []string{ + "shared_buffers = 0GB", + "work_mem = 0kB", + "effective_cache_size = 0GB", + "maintenance_work_mem = 0GB", + } +) + +func TestProcessSettingsGroup(t *testing.T) { + cases := []struct { + desc string + ts *tuneSettings + lines []string + stdin string + wantStatements uint64 + wantPrompts uint64 + wantPrints uint64 + wantErrors uint64 + successMsg string + shouldErr bool + }{ + { + desc: "no keys, no need to prompt", + ts: &tuneSettings{ + label: "foo", + rec: nil, + keys: nil, + }, + lines: memSettingsCorrect, + wantStatements: 1, // only intro remark + wantPrompts: 0, + wantPrints: 1, // one for initial newline + successMsg: "foo settings are already tuned", + shouldErr: false, + }, + { + desc: "memory - commented", + ts: &tuneSettings{ + label: "memory", + rec: &memoryRecommender{8 * gigabyte, 4}, + keys: memoryKeys, + }, + lines: memSettingsCommented, + stdin: "y\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 3, // one for initial newline + one setting, displayed twice + successMsg: "memory settings will be updated", + shouldErr: false, + }, + { + desc: "memory - wrong", + ts: &tuneSettings{ + label: "memory", + rec: &memoryRecommender{8 * gigabyte, 4}, + keys: memoryKeys, + }, + lines: memSettingsWrongVal, + stdin: "y\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 3, // one for initial newline + one setting, displayed twice + successMsg: "memory settings will be updated", + shouldErr: false, + }, + { + desc: "memory - missing", + ts: &tuneSettings{ + label: "memory", + rec: &memoryRecommender{8 * gigabyte, 4}, + keys: memoryKeys, + }, + lines: memSettingsMissing, + stdin: "y\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 2, // one for initial newline + one setting, displayed once (missing is now in printer.Error) + wantErrors: 1, // for missing + successMsg: "memory settings will be updated", + shouldErr: false, + }, + { + desc: "memory - comment+wrong", + ts: &tuneSettings{ + label: "memory", + rec: &memoryRecommender{8 * gigabyte, 4}, + keys: memoryKeys, + }, + lines: memSettingsCommentWrong, + stdin: " \ny\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 2, // first input is blank + wantPrints: 5, // one for initial newline + two settings, displayed twice + successMsg: "memory settings will be updated", + shouldErr: false, + }, + { + desc: "memory - comment+wrong+missing", + ts: &tuneSettings{ + label: "memory", + rec: &memoryRecommender{8 * gigabyte, 4}, + keys: memoryKeys, + }, + lines: memSettingsCommentWrongMissing, + stdin: " \n \ny\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 3, // first input is blank + wantPrints: 6, // one for initial newline + two settings, displayed twice, 1 setting once + wantErrors: 1, // for missing + successMsg: "memory settings will be updated", + shouldErr: false, + }, + { + desc: "memory - all wrong, but skip", + ts: &tuneSettings{ + label: "memory", + rec: &memoryRecommender{8 * gigabyte, 4}, + keys: memoryKeys, + }, + lines: memSettingsAllWrong, + stdin: "s\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 9, // one for initial newline + four settings, displayed twice + wantErrors: 1, + successMsg: "", + shouldErr: false, + }, + { + desc: "memory - all wrong, but quit", + ts: &tuneSettings{ + label: "memory", + rec: &memoryRecommender{8 * gigabyte, 4}, + keys: memoryKeys, + }, + lines: memSettingsAllWrong, + stdin: " \nqUIt\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 2, + wantPrints: 9, // one for initial newline + four settings, displayed twice + successMsg: "", + shouldErr: true, + }, + { + desc: "memory - all wrong", + ts: &tuneSettings{ + label: "memory", + rec: &memoryRecommender{8 * gigabyte, 4}, + keys: memoryKeys, + }, + lines: memSettingsAllWrong, + stdin: "y\n", + wantStatements: 3, // intro remark + current label + recommend label + wantPrompts: 1, + wantPrints: 9, // one for initial newline + four settings, displayed twice + successMsg: "memory settings will be updated", + shouldErr: false, + }, + { + desc: "label capitalized", + ts: &tuneSettings{ + label: "WAL", + rec: nil, + keys: []string{}, + }, + lines: []string{}, + stdin: "y\n", + wantStatements: 1, + wantPrints: 1, // one for initial newline + successMsg: "WAL settings are already tuned", + shouldErr: false, + }, + } + + oldPrintFn := printFn + + for _, c := range cases { + buf := bytes.NewBuffer([]byte(c.stdin)) + br := bufio.NewReader(buf) + handler := &ioHandler{ + p: &testPrinter{}, + br: br, + } + cfs := &configFileState{tuneParseResults: make(map[string]*tunableParseResult)} + cfs.lines = append(cfs.lines, c.lines...) + for i, l := range cfs.lines { + for _, k := range c.ts.keys { + p := parseWithRegex(l, regexes[k]) + if p != nil { + p.idx = i + cfs.tuneParseResults[k] = p + } + } + } + numPrints := uint64(0) + printFn = func(format string, args ...interface{}) (int, error) { + numPrints++ + return 0, nil + } + + err := processSettingsGroup(handler, cfs, c.ts, false) + if err != nil && !c.shouldErr { + t.Errorf("%s: unexpected error: %v", c.desc, err) + } else if err == nil && c.shouldErr { + t.Errorf("%s: unexpected lack of error", c.desc) + } + + tp := handler.p.(*testPrinter) + if got := strings.ToUpper(strings.TrimSpace(tp.statements[0])[:1]); got != strings.ToUpper(c.ts.label[:1]) { + t.Errorf("%s: label not capitalized in first statement: got %s want %s", c.desc, got, strings.ToUpper(c.ts.label[:1])) + } + + if got := tp.statementCalls; got != c.wantStatements { + t.Errorf("%s: incorrect number of statements: got %d want %d", c.desc, got, c.wantStatements) + } + + if got := tp.promptCalls; got != c.wantPrompts { + t.Errorf("%s: incorrect number of prompts: got %d want %d", c.desc, got, c.wantPrompts) + } + + if got := numPrints; got != c.wantPrints { + t.Errorf("%s: incorrect number of prints: got %d want %d", c.desc, got, c.wantPrints) + } + + if got := tp.errorCalls; got != c.wantErrors { + t.Errorf("%s: incorrect number of errors: got %d want %d", c.desc, got, c.wantErrors) + } else if len(c.successMsg) > 0 { + if got := tp.successCalls; got != 1 { + t.Errorf("%s: incorrect number of successes: got %d want %d", c.desc, got, 1) + } + if got := tp.successes[0]; got != c.successMsg { + t.Errorf("%s: incorrect success message: got\n%s\nwant\n%s\n", c.desc, got, c.successMsg) + } + } else if tp.successCalls > 0 { + t.Errorf("%s: got success without expecting it: %s", c.desc, tp.successes[0]) + } + } + + printFn = oldPrintFn +} + +func TestProcessTunables(t *testing.T) { + mem := uint64(10 * gigabyte) + cpus := 6 + oldPrintFn := printFn + printFn = func(_ string, _ ...interface{}) (int, error) { + return 0, nil + } + + buf := bytes.NewBuffer([]byte("y\ny\ny\ny\n")) + br := bufio.NewReader(buf) + handler := &ioHandler{ + p: &testPrinter{}, + br: br, + } + + cfs := &configFileState{lines: []string{}, tuneParseResults: make(map[string]*tunableParseResult)} + processTunables(handler, cfs, mem, cpus, false) + + tp := handler.p.(*testPrinter) + // Total number of statements is intro statement and then 3 per group of settings; + // each group has a heading and then the current/recommended labels. + if got := tp.statementCalls; got != 1+3*4 { + t.Errorf("incorrect number of statements: got %d, want %d", got, 1+3*4) + } + + wantStatement := fmt.Sprintf(statementTunableIntro, bytesFormat(mem), cpus) + if got := tp.statements[0]; got != wantStatement { + t.Errorf("incorrect first statement: got\n%s\nwant\n%s\n", got, wantStatement) + } + + for i := 2; i < len(tp.statements); i += 3 { + if got := tp.statements[i]; got != currentLabel { + t.Errorf("did not get current label as expected: got %s", got) + } + if got := tp.statements[i+1]; got != recommendLabel { + t.Errorf("did not get recommend label as expected: got %s", got) + } + } + + wantStatement = "Memory settings recommendations" + if got := tp.statements[1]; got != wantStatement { + t.Errorf("incorrect statement at 1: got\n%s\nwant\n%s", got, wantStatement) + } + wantStatement = "Parallelism settings recommendations" + if got := tp.statements[4]; got != wantStatement { + t.Errorf("incorrect statement at 4: got\n%s\nwant\n%s", got, wantStatement) + } + wantStatement = "WAL settings recommendations" + if got := tp.statements[7]; got != wantStatement { + t.Errorf("incorrect statement at 7: got\n%s\nwant\n%s", got, wantStatement) + } + wantStatement = "Miscellaneous settings recommendations" + if got := tp.statements[10]; got != wantStatement { + t.Errorf("incorrect statement at 10: got\n%s\nwant\n%s", got, wantStatement) + } + + printFn = oldPrintFn +} diff --git a/parse.go b/parse.go new file mode 100644 index 0000000..c7c098e --- /dev/null +++ b/parse.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + "math" + "regexp" + "strconv" +) + +const ( + terabyte = 1 << 40 + gigabyte = 1 << 30 + megabyte = 1 << 20 + kilobyte = 1 << 10 + tb = "TB" + gb = "GB" + mb = "MB" + kb = "kB" + b = "bytes" + + errIncorrectFormatFmt = "incorrect format for '%s'" +) + +var pgBytesRegex = regexp.MustCompile("^([0-9]+)((?:k|M|G|T)B)$") + +func parseIntToFloatUnits(bytes uint64) (float64, string) { + if bytes <= 0 { + panic("bytes must be at least 1 byte") + } + divisor := 1.0 + units := b + if bytes >= terabyte { + divisor = float64(terabyte) + units = tb + } else if bytes >= gigabyte { + divisor = float64(gigabyte) + units = gb + } else if bytes >= megabyte { + divisor = float64(megabyte) + units = mb + } else if bytes >= kilobyte { + divisor = float64(kilobyte) + units = kb + } + return float64(bytes) / divisor, units +} + +func bytesFormat(bytes uint64) string { + val, units := parseIntToFloatUnits(bytes) + return fmt.Sprintf("%0.2f %s", val, units) +} + +func bytesPGFormat(bytes uint64) string { + val, units := parseIntToFloatUnits(bytes) + if units == b { // nothing less than 1kB allowed + val = 1.0 + units = kb + } else if units == kb { + val = math.Round(val) + } else { + if val-float64(uint64(val)) > 0.001 { // (anything less than .001 is not going to meaningfully change at 1024x) + val = val * 1024 + if units == tb { + units = gb + } else if units == gb { + units = mb + } else if units == mb { + units = kb + } else { + panic(fmt.Sprintf("unknown units: %s", units)) + } + } + } + return fmt.Sprintf("%d%s", uint64(val), units) +} + +func parsePGStringToBytes(val string) (float64, error) { + res := pgBytesRegex.FindStringSubmatch(val) + if len(res) != 3 { + return 0.0, fmt.Errorf(errIncorrectFormatFmt, val) + } + num, err := strconv.ParseInt(res[1], 10, 64) + if err != nil { + return 0.0, fmt.Errorf("could not parse bytes number: %v", err) + } + units := res[2] + var ret uint64 + if units == kb { + ret = uint64(num) * kilobyte + } else if units == mb { + ret = uint64(num) * megabyte + } else if units == gb { + ret = uint64(num) * gigabyte + } else if units == tb { + ret = uint64(num) * terabyte + } else { + return 0, fmt.Errorf("unknown units: %s", units) + } + return float64(ret), nil +} diff --git a/parse_test.go b/parse_test.go new file mode 100644 index 0000000..7160e3d --- /dev/null +++ b/parse_test.go @@ -0,0 +1,305 @@ +package main + +import ( + "fmt" + "testing" +) + +func TestParseIntToFloatUnits(t *testing.T) { + cases := []struct { + desc string + input uint64 + wantNum float64 + wantUnits string + }{ + { + desc: "no limit to TB", + input: 2000 * terabyte, + wantNum: 2000, + wantUnits: tb, + }, + { + desc: "1 TB", + input: terabyte, + wantNum: 1.0, + wantUnits: tb, + }, + { + desc: "1.5 TB", + input: uint64(1.5 * float64(terabyte)), + wantNum: 1.5, + wantUnits: tb, + }, + { + desc: "1TB - 1GB", + input: terabyte - gigabyte, + wantNum: 1023, + wantUnits: gb, + }, + { + desc: "1 GB", + input: gigabyte, + wantNum: 1.0, + wantUnits: gb, + }, + { + desc: "1.5 GB", + input: uint64(1.5 * float64(gigabyte)), + wantNum: 1.5, + wantUnits: gb, + }, + { + desc: "2.0 GB", + input: 2 * gigabyte, + wantNum: 2.0, + wantUnits: gb, + }, + { + desc: "1 GB - 1 MB", + input: gigabyte - megabyte, + wantNum: 1023.0, + wantUnits: mb, + }, + { + desc: "1 MB", + input: megabyte, + wantNum: 1.0, + wantUnits: mb, + }, + { + desc: "1.5 MB", + input: uint64(1.5 * float64(megabyte)), + wantNum: 1.5, + wantUnits: mb, + }, + { + desc: "1020 kB", + input: megabyte - 4*kilobyte, + wantNum: 1020.0, + wantUnits: kb, + }, + { + desc: "1 kB", + input: kilobyte, + wantNum: 1.0, + wantUnits: kb, + }, + { + desc: "1.5 kB", + input: uint64(1.5 * float64(kilobyte)), + wantNum: 1.5, + wantUnits: kb, + }, + { + desc: "1000 bytes", + input: kilobyte - 24, + wantNum: 1000, + wantUnits: b, + }, + } + + for _, c := range cases { + val, units := parseIntToFloatUnits(c.input) + if got := val; got != c.wantNum { + t.Errorf("%s: incorrect val: got %f want %f", c.desc, got, c.wantNum) + } + if got := units; got != c.wantUnits { + t.Errorf("%s: incorrect units: got %s want %s", c.desc, got, c.wantUnits) + } + } +} + +func TestParseIntToFloatUnitsPanic(t *testing.T) { + func() { + defer func() { + if re := recover(); re == nil { + t.Errorf("did not panic when should") + } + }() + parseIntToFloatUnits(0) + }() +} + +func TestBytesPGFormat(t *testing.T) { + cases := []struct { + desc string + input uint64 + want string + }{ + { + desc: "no limit to TB", + input: 2000 * terabyte, + want: "2000" + tb, + }, + { + desc: "1 TB", + input: terabyte, + want: "1" + tb, + }, + { + desc: "1.5 TB", + input: uint64(1.5 * float64(terabyte)), + want: "1536" + gb, + }, + { + desc: "1TB - 1GB", + input: terabyte - gigabyte, + want: "1023" + gb, + }, + { + desc: "1TB - 1MB", + input: terabyte - megabyte, + want: "1048575" + mb, + }, + { + desc: "1 GB", + input: gigabyte, + want: "1" + gb, + }, + { + desc: "1.5 GB", + input: uint64(1.5 * float64(gigabyte)), + want: "1536" + mb, + }, + { + desc: "2.0 GB", + input: 2 * gigabyte, + want: "2" + gb, + }, + { + desc: "1 GB - 1MB", + input: gigabyte - megabyte, + want: "1023" + mb, + }, + { + desc: "1 MB", + input: megabyte, + want: "1" + mb, + }, + { + desc: "1.5 MB", + input: uint64(1.5 * float64(megabyte)), + want: "1536" + kb, + }, + { + desc: "1020 kB", + input: megabyte - 4*kilobyte, + want: "1020" + kb, + }, + { + desc: "1 kB", + input: kilobyte, + want: "1" + kb, + }, + { + desc: "1.5 kB, round up", + input: uint64(1.5 * float64(kilobyte)), + want: "2" + kb, + }, + { + desc: "1.4 kB, round down", + input: 1400, + want: "1" + kb, + }, + { + desc: "1000 bytes", + input: kilobyte - 24, + want: "1" + kb, + }, + } + + for _, c := range cases { + if got := bytesPGFormat(c.input); got != c.want { + t.Errorf("%s: incorrect return: got %s want %s", c.desc, got, c.want) + } + } +} + +func TestParsePGStringToBytes(t *testing.T) { + cases := []struct { + desc string + input string + want float64 + errMsg string + }{ + { + desc: "incorrect format #1", + input: " 64MB", // no leading spaces + errMsg: fmt.Sprintf(errIncorrectFormatFmt, " 64MB"), + }, + { + desc: "incorrect format #2", + input: "64b", // bytes not allowed + errMsg: fmt.Sprintf(errIncorrectFormatFmt, "64b"), + }, + { + desc: "incorrect format #3", + input: "64 GB", // no space between num and units, + errMsg: fmt.Sprintf(errIncorrectFormatFmt, "64 GB"), + }, + { + desc: "incorrect format #4", + input: "-64MB", // negative memory is a no-no + errMsg: fmt.Sprintf(errIncorrectFormatFmt, "-64MB"), + }, + { + desc: "valid kilobytes", + input: "64" + kb, + want: 64 * kilobyte, + }, + { + desc: "valid kilobytes, oversized", + input: "2048" + kb, + want: 2048 * kilobyte, + }, + { + desc: "valid megabytes", + input: "64" + mb, + want: 64 * megabyte, + }, + { + desc: "valid megabytes, oversized", + input: "2048" + mb, + want: 2048 * megabyte, + }, + { + desc: "valid gigabytes", + input: "64" + gb, + want: 64 * gigabyte, + }, + { + desc: "valid gigabytes, oversized", + input: "2048" + gb, + want: 2048 * gigabyte, + }, + { + desc: "valid terabytes", + input: "64" + tb, + want: 64 * terabyte, + }, + { + desc: "valid terabytes, oversized", + input: "2048" + tb, + want: 2048 * terabyte, + }, + } + + for _, c := range cases { + bytes, err := parsePGStringToBytes(c.input) + if len(c.errMsg) > 0 { // failure cases + if err == nil { + t.Errorf("%s: unexpectedly err is nil: want %s", c.desc, c.errMsg) + } else if got := err.Error(); got != c.errMsg { + t.Errorf("%s: unexpected err msg: got\n%s\nwant\n%s", c.desc, got, c.errMsg) + } + } else { + if err != nil { + t.Errorf("%s: unexpected err: got %v", c.desc, err) + } + if got := bytes; got != c.want { + t.Errorf("%s: incorrect bytes: got %f want %f", c.desc, got, c.want) + } + } + + } +} diff --git a/print.go b/print.go new file mode 100644 index 0000000..8e97961 --- /dev/null +++ b/print.go @@ -0,0 +1,65 @@ +package main + +import ( + "strings" + + "github.com/fatih/color" +) + +const successLabel = "success: " + +var ( + statementColor = color.New(color.FgWhite, color.Bold) // color for directions / statements + promptColor = color.New(color.FgMagenta, color.Bold) // color for prompt/questions requiring user input + successColor = color.New(color.FgGreen, color.Bold) + errorColor = color.New(color.FgRed, color.Bold) +) + +type printer interface { + Statement(string, ...interface{}) + Prompt(string, ...interface{}) + Success(string, ...interface{}) + Error(string, string, ...interface{}) +} + +type colorPrinter struct{} + +func printWithColor(c *color.Color, format string, args ...interface{}) { + c.Printf(format, args...) +} + +func (p *colorPrinter) Statement(format string, args ...interface{}) { + printWithColor(statementColor, format+"\n", args...) +} + +func (p *colorPrinter) Prompt(format string, args ...interface{}) { + printWithColor(promptColor, format, args...) +} + +func (p *colorPrinter) Success(format string, args ...interface{}) { + printWithColor(successColor, successLabel) + printFn(format+"\n", args...) +} + +func (p *colorPrinter) Error(label string, format string, args ...interface{}) { + printWithColor(errorColor, label+": ") + printFn(format+"\n", args...) +} + +type noColorPrinter struct{} + +func (p *noColorPrinter) Statement(format string, args ...interface{}) { + printFn("== "+format+"\n", args...) +} + +func (p *noColorPrinter) Prompt(format string, args ...interface{}) { + printFn("-- "+format, args...) +} + +func (p *noColorPrinter) Success(format string, args ...interface{}) { + printFn(strings.ToUpper(successLabel)+format+"\n", args...) +} + +func (p *noColorPrinter) Error(label string, format string, args ...interface{}) { + printFn(strings.ToUpper(label)+": "+format+"\n", args...) +} diff --git a/print_test.go b/print_test.go new file mode 100644 index 0000000..5ffee88 --- /dev/null +++ b/print_test.go @@ -0,0 +1,34 @@ +package main + +import "fmt" + +type testPrinter struct { + statementCalls uint64 + statements []string + promptCalls uint64 + prompts []string + successCalls uint64 + successes []string + errorCalls uint64 + errors []string +} + +func (p *testPrinter) Statement(format string, args ...interface{}) { + p.statementCalls++ + p.statements = append(p.statements, fmt.Sprintf(format, args...)) +} + +func (p *testPrinter) Prompt(format string, args ...interface{}) { + p.promptCalls++ + p.prompts = append(p.prompts, fmt.Sprintf(format, args...)) +} + +func (p *testPrinter) Success(format string, args ...interface{}) { + p.successCalls++ + p.successes = append(p.successes, fmt.Sprintf(format, args...)) +} + +func (p *testPrinter) Error(_ string, format string, args ...interface{}) { + p.errorCalls++ + p.errors = append(p.errors, fmt.Sprintf(format, args...)) +} diff --git a/tune.go b/tune.go new file mode 100644 index 0000000..9bf8c29 --- /dev/null +++ b/tune.go @@ -0,0 +1,95 @@ +package main + +import ( + "fmt" + "math" + "regexp" + "runtime" + "strconv" +) + +const ( + regexFmt = "^(\\s*#+?\\s*)?(%s) = (\\S+?)(\\s*(?:#.*|))$" + osWindows = "windows" +) + +var ( + regexes = make(map[string]*regexp.Regexp) + parsers = make(map[string]parseFn) +) + +type tunableParseResult struct { + idx int + commented bool + missing bool + key string + value string + extra string +} + +type recommender interface { + Recommend(string) string +} + +func keyToRegex(key string) *regexp.Regexp { + return regexp.MustCompile(fmt.Sprintf(regexFmt, key)) +} + +func isIn(key string, arr []string) bool { + for _, s := range arr { + if key == s { + return true + } + } + return false +} + +type parseFn func(string) (float64, error) + +func keyToParseFn(key string) parseFn { + if isIn(key, memoryKeys) || isIn(key, walKeys) { + return parsePGStringToBytes + } + + return func(s string) (float64, error) { + return strconv.ParseFloat(s, 64) + } +} + +func init() { + setup := func(arr []string) { + for _, k := range arr { + regexes[k] = keyToRegex(k) + parsers[k] = keyToParseFn(k) + } + } + if runtime.GOOS == osWindows { + otherKeys = otherKeys[:len(otherKeys)-1] + } + + setup(memoryKeys) + setup(parallelKeys) + setup(walKeys) + setup(otherKeys) +} + +func parseWithRegex(line string, regex *regexp.Regexp) *tunableParseResult { + res := regex.FindStringSubmatch(line) + if len(res) > 0 { + if len(res) != 5 { + panic(fmt.Sprintf("unexpected regex parse result: %v (len = %d)", res, len(res))) + } + + return &tunableParseResult{ + commented: len(res[1]) > 0, + key: res[2], + value: res[3], + extra: res[4], + } + } + return nil +} + +func isCloseEnough(actual, target, fudge float64) bool { + return math.Abs((target-actual)/target) <= fudge +} diff --git a/tune_memory.go b/tune_memory.go new file mode 100644 index 0000000..d52b340 --- /dev/null +++ b/tune_memory.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + "math" + "runtime" +) + +const ( + sharedBuffersKey = "shared_buffers" + effectiveCacheKey = "effective_cache_size" + maintenanceWorkMemKey = "maintenance_work_mem" + workMemKey = "work_mem" + + sharedBuffersWindows = 512 * megabyte +) + +var memoryKeys = []string{ + sharedBuffersKey, + effectiveCacheKey, + maintenanceWorkMemKey, + workMemKey, +} + +type memoryRecommender struct { + totalMem uint64 + cpus int +} + +func (r *memoryRecommender) Recommend(key string) string { + var val string + if key == sharedBuffersKey { + if runtime.GOOS == osWindows { + val = bytesPGFormat(sharedBuffersWindows) + } else { + val = bytesPGFormat(r.totalMem / 4) + } + } else if key == effectiveCacheKey { + val = bytesPGFormat((r.totalMem * 3) / 4) + } else if key == maintenanceWorkMemKey { + temp := (float64(r.totalMem) / float64(gigabyte)) * (128.0 * float64(megabyte)) + if temp > (2 * gigabyte) { + temp = 2 * gigabyte + } + val = bytesPGFormat(uint64(temp)) + } else if key == workMemKey { + if runtime.GOOS == osWindows { + val = r.recommendWindows() + } else { + cpuFactor := math.Round(float64(r.cpus) / 2.0) + temp := (float64(r.totalMem) / float64(gigabyte)) * (6.4 * float64(megabyte)) / cpuFactor + val = bytesPGFormat(uint64(temp)) + } + } else { + panic(fmt.Sprintf("unknown key: %s", key)) + } + return val +} + +func (r *memoryRecommender) recommendWindows() string { + cpuFactor := math.Round(float64(r.cpus) / 2.0) + if r.totalMem <= 2*gigabyte { + temp := (float64(r.totalMem) / float64(gigabyte)) * (6.4 * float64(megabyte)) / cpuFactor + return bytesPGFormat(uint64(temp)) + } + base := 2.0 * 6.4 * float64(megabyte) + temp := ((float64(r.totalMem)/float64(gigabyte)-2)*(8.53336*float64(megabyte)) + base) / cpuFactor + return bytesPGFormat(uint64(temp)) +} diff --git a/tune_memory_test.go b/tune_memory_test.go new file mode 100644 index 0000000..2ee4424 --- /dev/null +++ b/tune_memory_test.go @@ -0,0 +1,171 @@ +package main + +import ( + "fmt" + "testing" +) + +func TestMemoryRecommenderRecommendWindows(t *testing.T) { + cases := []struct { + desc string + totalMem uint64 + cpus int + want string + }{ + { + desc: "1GB", + totalMem: 1 * gigabyte, + cpus: 1, + want: "6553" + kb, // from pgtune + }, + { + desc: "1GB, 4 cpus", + totalMem: 1 * gigabyte, + cpus: 4, + want: "3276" + kb, // from pgtune + }, + { + desc: "2GB", + totalMem: 2 * gigabyte, + cpus: 1, + want: "13107" + kb, // from pgtune + }, + { + desc: "2GB, 5 cpus", + totalMem: 2 * gigabyte, + cpus: 5, + want: "4369" + kb, // from pgtune + }, + { + desc: "3GB", + totalMem: 3 * gigabyte, + cpus: 1, + want: "21845" + kb, // from pgtune + }, + { + desc: "3GB, 3 cpus", + totalMem: 3 * gigabyte, + cpus: 3, + want: "10922" + kb, // from pgtune + }, + { + desc: "8GB", + totalMem: 8 * gigabyte, + cpus: 1, + want: "64" + mb, // from pgtune + }, + { + desc: "8GB, 8 cpus", + totalMem: 8 * gigabyte, + cpus: 8, + want: "16" + mb, // from pgtune + }, + { + desc: "16GB", + totalMem: 16 * gigabyte, + cpus: 1, + want: "135441" + kb, // from pgtune + }, + { + desc: "16GB, 10 cpus", + totalMem: 16 * gigabyte, + cpus: 10, + want: "27088" + kb, // from pgtune + }, + } + + for _, c := range cases { + mr := &memoryRecommender{c.totalMem, c.cpus} + if got := mr.recommendWindows(); got != c.want { + t.Errorf("%s: incorrect value: got %s want %s", c.desc, got, c.want) + } + } +} + +func TestMemoryRecommenderRecommend(t *testing.T) { + valFmt := "%d%s" + cases := []struct { + desc string + key string + totalMem uint64 + cpus int + want string + }{ + { + desc: "shared_buffers, uneven divide", + key: sharedBuffersKey, + totalMem: 10 * gigabyte, + cpus: 1, + want: fmt.Sprintf(valFmt, 2560, mb), + }, + { + desc: "shared_buffers, even divide", + key: sharedBuffersKey, + totalMem: 8 * gigabyte, + cpus: 1, + want: fmt.Sprintf(valFmt, 2, gb), + }, + { + desc: "effective key, uneven divide", + key: effectiveCacheKey, + totalMem: 10 * gigabyte, + cpus: 1, + want: fmt.Sprintf(valFmt, uint64(7.5*1024.0), mb), + }, + { + desc: "effective key, even divide", + key: effectiveCacheKey, + totalMem: 12 * gigabyte, + cpus: 1, + want: fmt.Sprintf(valFmt, 9, gb), + }, + { + desc: "maintenance_work_mem", + key: maintenanceWorkMemKey, + totalMem: 6 * gigabyte, + cpus: 1, + want: fmt.Sprintf(valFmt, 768, mb), + }, + { + desc: "maintenance_work_mem, over max", + key: maintenanceWorkMemKey, + totalMem: 32 * gigabyte, + cpus: 1, + want: fmt.Sprintf(valFmt, 2, gb), + }, + { + desc: "work_mem", + key: workMemKey, + totalMem: 8 * gigabyte, + cpus: 1, + want: fmt.Sprintf(valFmt, 52428, kb), + }, + { + desc: "work_mem, multiple CPUs", + key: workMemKey, + totalMem: 8 * gigabyte, + cpus: 4, + want: fmt.Sprintf(valFmt, 26214, kb), + }, + } + + for _, c := range cases { + mr := &memoryRecommender{c.totalMem, c.cpus} + got := mr.Recommend(c.key) + if got != c.want { + t.Fatalf("%s: incorrect result: got\n%s\nwant\n%s", c.desc, got, c.want) + } + } +} + +func TestMemoryRecommenderRecommendPanic(t *testing.T) { + func() { + r := &memoryRecommender{1, 1} + defer func() { + if re := recover(); re == nil { + t.Errorf("did not panic when should") + } + }() + r.Recommend("foo") + }() +} diff --git a/tune_misc.go b/tune_misc.go new file mode 100644 index 0000000..bbd0acf --- /dev/null +++ b/tune_misc.go @@ -0,0 +1,46 @@ +package main + +import "fmt" + +const ( + checkpointKey = "checkpoint_completion_target" + statsTargetKey = "default_statistics_target" + maxConnectionsKey = "max_connections" + randomPageCostKey = "random_page_cost" + // linux only + effectiveIOKey = "effective_io_concurrency" + + checkpointDefault = "0.9" + statsTargetDefault = "500" + maxConnectionsDefault = "20" + randomPageCostDefault = "1.1" + effectiveIODefault = "200" +) + +var otherKeys = []string{ + statsTargetKey, + randomPageCostKey, + effectiveIOKey, + checkpointKey, + maxConnectionsKey, +} + +type miscRecommender struct{} + +func (r *miscRecommender) Recommend(key string) string { + var val string + if key == checkpointKey { + val = checkpointDefault + } else if key == statsTargetKey { + val = statsTargetDefault + } else if key == maxConnectionsKey { + val = maxConnectionsDefault + } else if key == randomPageCostKey { + val = randomPageCostDefault + } else if key == effectiveIOKey { + val = effectiveIODefault + } else { + panic(fmt.Sprintf("unknown key: %s", key)) + } + return val +} diff --git a/tune_misc_test.go b/tune_misc_test.go new file mode 100644 index 0000000..b9ceb5f --- /dev/null +++ b/tune_misc_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "testing" +) + +func TestMiscRecommenderRecommend(t *testing.T) { + cases := []struct { + desc string + key string + want string + }{ + { + desc: checkpointKey, + key: checkpointKey, + want: checkpointDefault, + }, + { + desc: statsTargetKey, + key: statsTargetKey, + want: statsTargetDefault, + }, + { + desc: maxConnectionsKey, + key: maxConnectionsKey, + want: maxConnectionsDefault, + }, + { + desc: randomPageCostKey, + key: randomPageCostKey, + want: randomPageCostDefault, + }, + { + desc: effectiveIOKey, + key: effectiveIOKey, + want: effectiveIODefault, + }, + } + + for _, c := range cases { + r := &miscRecommender{} + got := r.Recommend(c.key) + if got != c.want { + t.Errorf("%s: incorrect result: got\n%s\nwant\n%s", c.desc, got, c.want) + } + } +} + +func TestMiscRecommenderRecommendPanic(t *testing.T) { + func() { + r := &miscRecommender{} + defer func() { + if re := recover(); re == nil { + t.Errorf("did not panic when should") + } + }() + r.Recommend("foo") + }() +} diff --git a/tune_parallel.go b/tune_parallel.go new file mode 100644 index 0000000..870e4a3 --- /dev/null +++ b/tune_parallel.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "math" +) + +const ( + maxWorkerProcessesKey = "max_worker_processes" + maxParallelWorkersGatherKey = "max_parallel_workers_per_gather" + maxParallelWorkers = "max_parallel_workers" + + errOneCPU = "cannot make recommendations with just 1 CPU" +) + +var parallelKeys = []string{ + maxWorkerProcessesKey, + maxParallelWorkersGatherKey, + maxParallelWorkers, +} + +type parallelRecommender struct { + cpus int +} + +func (r *parallelRecommender) Recommend(key string) string { + var val string + if r.cpus <= 1 { + panic(errOneCPU) + } + if key == maxWorkerProcessesKey || key == maxParallelWorkers { + val = fmt.Sprintf("%d", r.cpus) + } else if key == maxParallelWorkersGatherKey { + val = fmt.Sprintf("%d", int(math.Round(float64(r.cpus)/2.0))) + } else { + panic(fmt.Sprintf("unknown key: %s", key)) + } + return val +} diff --git a/tune_parallel_test.go b/tune_parallel_test.go new file mode 100644 index 0000000..ce12527 --- /dev/null +++ b/tune_parallel_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "testing" +) + +func TestParallelRecommenderRecommend(t *testing.T) { + cases := []struct { + desc string + key string + cpus int + want string + }{ + { + desc: "max_worker_processes, 2", + key: maxWorkerProcessesKey, + cpus: 2, + want: "2", + }, + { + desc: "max_worker_processes, 4", + key: maxWorkerProcessesKey, + cpus: 4, + want: "4", + }, + { + desc: "max_worker_processes, 5", + key: maxWorkerProcessesKey, + cpus: 5, + want: "5", + }, + { + desc: "max_parallel_workers, 2", + key: maxParallelWorkers, + cpus: 2, + want: "2", + }, + { + desc: "max_parallel_workers, 4", + key: maxParallelWorkers, + cpus: 4, + want: "4", + }, + { + desc: "max_parallel_workers, 5", + key: maxParallelWorkers, + cpus: 5, + want: "5", + }, + { + desc: "max_parallel_workers_per_gather, 2", + key: maxParallelWorkersGatherKey, + cpus: 2, + want: "1", + }, + { + desc: "max_parallel_workers_per_gather, 4", + key: maxParallelWorkersGatherKey, + cpus: 4, + want: "2", + }, + { + desc: "max_parallel_workers_per_gather, 5", + key: maxParallelWorkersGatherKey, + cpus: 5, + want: "3", + }, + } + + for _, c := range cases { + r := ¶llelRecommender{c.cpus} + got := r.Recommend(c.key) + if got != c.want { + t.Errorf("%s: incorrect result: got\n%s\nwant\n%s", c.desc, got, c.want) + } + } +} + +func TestParallelRecommenderRecommendPanics(t *testing.T) { + func() { + r := ¶llelRecommender{5} + defer func() { + if re := recover(); re == nil { + t.Errorf("did not panic when should") + } + }() + r.Recommend("foo") + }() + + func() { + r := ¶llelRecommender{1} + defer func() { + if re := recover(); re == nil { + t.Errorf("did not panic when should") + } + }() + r.Recommend("foo") + }() +} diff --git a/tune_test.go b/tune_test.go new file mode 100644 index 0000000..4e4aa8d --- /dev/null +++ b/tune_test.go @@ -0,0 +1,218 @@ +package main + +import ( + "testing" +) + +const testKey = "test_setting" + +var testRegex = keyToRegex(testKey) + +func TestIsIn(t *testing.T) { + cases := []struct { + desc string + key string + arr []string + want bool + }{ + { + desc: "yes, len 1", + key: "foo", + arr: []string{"foo"}, + want: true, + }, + { + desc: "no, len 0", + key: "foo", + arr: []string{}, + want: false, + }, + { + desc: "no, len 1", + key: "bar", + arr: []string{"foo"}, + want: false, + }, + { + desc: "no, len 3", + key: "bar", + arr: []string{"foo1", "foo2", "foo3"}, + want: false, + }, + { + desc: "yes, len 3", + key: "foo2", + arr: []string{"foo1", "foo2", "foo3"}, + want: true, + }, + } + + for _, c := range cases { + if got := isIn(c.key, c.arr); got != c.want { + t.Errorf("%s: incorrect value: got %v want %v", c.desc, got, c.want) + } + } +} + +func TestKeyToParseFn(t *testing.T) { + cases := []struct { + desc string + key string + parseInput string + want float64 + }{ + { + desc: "memory key", + key: memoryKeys[0], + parseInput: "10" + gb, + want: float64(10 * gigabyte), + }, + { + desc: "wal key", + key: walKeys[0], + parseInput: "5" + mb, + want: float64(5 * megabyte), + }, + { + desc: "other key", + key: otherKeys[0], + parseInput: "501.0", + want: 501.0, + }, + } + + for _, c := range cases { + got, err := keyToParseFn(c.key)(c.parseInput) + if err != nil { + t.Errorf("%s: unexpected error: %v", c.desc, err) + } else if got != c.want { + t.Errorf("%s: incorrect result: got %v want %v", c.desc, got, c.want) + } + } +} + +func TestParseWithRegex(t *testing.T) { + cases := []struct { + desc string + input string + want *tunableParseResult + }{ + { + desc: "simple correct", + input: testKey + " = 50.0", + want: &tunableParseResult{ + commented: false, + key: testKey, + value: "50.0", + extra: "", + }, + }, + { + desc: "correct, comment at end", + input: testKey + " = 50.0 # do not change!", + want: &tunableParseResult{ + commented: false, + key: testKey, + value: "50.0", + extra: " # do not change!", + }, + }, + { + desc: "correct, comment at end no space", + input: testKey + " = 50.0# do not change!", + want: &tunableParseResult{ + commented: false, + key: testKey, + value: "50.0", + extra: "# do not change!", + }, + }, + { + desc: "correct, comment at end more space", + input: testKey + " = 50.0 # do not change!", + want: &tunableParseResult{ + commented: false, + key: testKey, + value: "50.0", + extra: " # do not change!", + }, + }, + { + desc: "correct, comment at end tabs", + input: testKey + " = 50.0 # do not change!", + want: &tunableParseResult{ + commented: false, + key: testKey, + value: "50.0", + extra: " # do not change!", + }, + }, + { + desc: "correct, tabs at the end", + input: testKey + " = 50.0 ", + want: &tunableParseResult{ + commented: false, + key: testKey, + value: "50.0", + extra: " ", + }, + }, + { + desc: "simple correct, commented", + input: "#" + testKey + " = 50.0", + want: &tunableParseResult{ + commented: true, + key: testKey, + value: "50.0", + extra: "", + }, + }, + { + desc: "commented with spaces", + input: " # " + testKey + " = 50.0", + want: &tunableParseResult{ + commented: true, + key: testKey, + value: "50.0", + extra: "", + }, + }, + { + desc: "commented with ending comment", + input: "# " + testKey + " = 50.0 # do not change", + want: &tunableParseResult{ + commented: true, + key: testKey, + value: "50.0", + extra: " # do not change", + }, + }, + { + desc: "incorrect, do not accept comments with starting #", + input: testKey + " = 50.0 do not change!", + want: nil, + }, + } + + for _, c := range cases { + res := parseWithRegex(c.input, testRegex) + if res == nil && c.want != nil { + t.Errorf("%s: result was unexpectedly nil: want %v", c.desc, c.want) + } else if res != nil && c.want == nil { + t.Errorf("%s: result was unexpectedly non-nil: got %v", c.desc, res) + } else if c.want != nil { + if got := res.commented; got != c.want.commented { + t.Errorf("%s: incorrect commented: got %v want %v", c.desc, got, c.want.commented) + } + if got := res.key; got != c.want.key { + t.Errorf("%s: incorrect key: got %v want %v", c.desc, got, c.want.key) + } + if got := res.value; got != c.want.value { + t.Errorf("%s: incorrect value: got %s want %s", c.desc, got, c.want.value) + } + if got := res.extra; got != c.want.extra { + t.Errorf("%s: incorrect extra: got %s want %s", c.desc, got, c.want.extra) + } + } + } +} diff --git a/tune_wal.go b/tune_wal.go new file mode 100644 index 0000000..5a4edae --- /dev/null +++ b/tune_wal.go @@ -0,0 +1,43 @@ +package main + +import "fmt" + +const ( + walBuffersKey = "wal_buffers" + minWalKey = "min_wal_size" + maxWalKey = "max_wal_size" + + walBuffersThreshold = 2 * gigabyte + walBuffersDefault = 16 * megabyte + minWalBytes = 4 * gigabyte + maxWalBytes = 8 * gigabyte +) + +var walKeys = []string{ + walBuffersKey, + minWalKey, + maxWalKey, +} + +type walRecommender struct { + totalMem uint64 +} + +func (r *walRecommender) Recommend(key string) string { + var val string + if key == walBuffersKey { + if r.totalMem < walBuffersThreshold { + temp := (float64(r.totalMem) / float64(gigabyte)) * (7864.0 * float64(kilobyte)) + val = bytesPGFormat(uint64(temp)) + } else { + val = bytesPGFormat(walBuffersDefault) + } + } else if key == minWalKey { + val = bytesPGFormat(minWalBytes) + } else if key == maxWalKey { + val = bytesPGFormat(maxWalBytes) + } else { + panic(fmt.Sprintf("unknown key: %s", key)) + } + return val +} diff --git a/tune_wal_test.go b/tune_wal_test.go new file mode 100644 index 0000000..f939036 --- /dev/null +++ b/tune_wal_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + "testing" +) + +func TestWALRecommenderRecommend(t *testing.T) { + valFmt := "%d%s" + cases := []struct { + desc string + key string + totalMem uint64 + want string + }{ + { + desc: "wal_buffers, 1GB", + key: walBuffersKey, + totalMem: 1 * gigabyte, + want: fmt.Sprintf(valFmt, 7864, kb), // from pgtune + }, + { + desc: "wal_buffers, 1.5GB", + key: walBuffersKey, + totalMem: uint64(1.5 * float64(gigabyte)), + want: fmt.Sprintf(valFmt, 11796, kb), // from pgtune + }, + { + desc: "wal_buffers, 2GB", + key: walBuffersKey, + totalMem: 2 * gigabyte, + want: fmt.Sprintf(valFmt, 16, mb), + }, + { + desc: "wal_buffers, > 2GB", + key: walBuffersKey, + totalMem: 10 * gigabyte, + want: fmt.Sprintf(valFmt, walBuffersDefault/megabyte, mb), + }, + { + desc: "min_wal_size is constant #1", + key: minWalKey, + totalMem: 1 * gigabyte, + want: fmt.Sprintf(valFmt, minWalBytes/gigabyte, gb), + }, + { + desc: "min_wal_size is constant #2", + key: minWalKey, + totalMem: 10 * gigabyte, + want: fmt.Sprintf(valFmt, minWalBytes/gigabyte, gb), + }, + { + desc: "max_wal_size is constant #1", + key: maxWalKey, + totalMem: 1 * gigabyte, + want: fmt.Sprintf(valFmt, maxWalBytes/gigabyte, gb), + }, + { + desc: "max_wal_size is constant #2", + key: maxWalKey, + totalMem: 10 * gigabyte, + want: fmt.Sprintf(valFmt, maxWalBytes/gigabyte, gb), + }, + } + + for _, c := range cases { + r := &walRecommender{c.totalMem} + got := r.Recommend(c.key) + if got != c.want { + t.Errorf("%s: incorrect result: got\n%s\nwant\n%s", c.desc, got, c.want) + } + } +} + +func TestWALRecommenderRecommendPanic(t *testing.T) { + func() { + r := &walRecommender{0} + defer func() { + if re := recover(); re == nil { + t.Errorf("did not panic when should") + } + }() + r.Recommend("foo") + }() +}