pulumi/pkg/backend/httpstate/client/marshal_pre_1.20.go

130 lines
3.7 KiB
Go
Raw Normal View History

[snapshot] Use a newer diff package The version of gopls that gotextdiff is derived from is over three years old. In the intervening time, the differ used by gopls has improved tremendously, and the language has grown new features like generics. These changes use a different diff package that is derived from an up-to-date version of gopls and exposes an API that is parameterized over the storage of the text. Taken together, this allows us to use a much faster and more memory-efficient algorithm for diffing while safely avoiding string copies when invoking the differ. The more efficient algorithm operates on arbitrary slices whose elements are comparable. Naive use of this algorithm--passing byte slices in--can cause increased time spent diffing, as the sequences being compared are longer. In order to avoid this, these changes record spans of marshaled deployments as the deployments are marshaled. These spans are strings that cover the diffable elements of the deployment: its header, its resources, and its pending operations. The output of the newer algorithm is converible to the format we currently send to the service, so these changes should not require service-side changes. In order to avoid bumping the minimum Go version required to build the `github.com/pulumi/pulumi/pkg/v3" and therefore break backwards compatibility with consumers using older toolchains (e.g. providers), these changes use the existing differ when built with a pre-1.20 toolchain. Official builds of the CLI are already using Go 1.20+ and should use the new differ without additional changes. These changes improve CPU time, allocation volume, and the compression ratio by 57%, 87%, and 52%, respectively. Benchmark results and analysis: httpstate ❯ go test -count=10 -run none -benchmem -bench . | tee spanned.txt httpstate ❯ benchstat buffer-pool.txt spanned.txt >stat.txt ``` │ buffer-pool.txt │ spanned.txt │ │ sec/op │ sec/op vs base │ DiffStack/1_x_2_B-10 45.09µ ± 1% 22.81µ ± 1% -49.41% (p=0.000 n=10) DiffStack/2_x_2_B-10 79.49µ ± 1% 40.60µ ± 1% -48.93% (p=0.000 n=10) DiffStack/4_x_2_B-10 172.54µ ± 2% 81.57µ ± 0% -52.72% (p=0.000 n=10) DiffStack/8_x_2_B-10 455.4µ ± 1% 200.4µ ± 0% -56.00% (p=0.000 n=10) DiffStack/16_x_2_B-10 1388.2µ ± 1% 575.1µ ± 1% -58.57% (p=0.000 n=10) DiffStack/32_x_2_B-10 4.409m ± 1% 1.866m ± 0% -57.67% (p=0.000 n=10) DiffStack/48_x_2_B-10 9.275m ± 0% 3.484m ± 0% -62.44% (p=0.000 n=10) DiffStack/64_x_2_B-10 14.564m ± 0% 5.945m ± 1% -59.18% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 124.81µ ± 1% 40.14µ ± 1% -67.84% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 299.8µ ± 0% 121.8µ ± 1% -59.38% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 846.2µ ± 1% 355.8µ ± 0% -57.96% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 2.743m ± 1% 1.150m ± 0% -58.06% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 9.271m ± 1% 4.041m ± 0% -56.41% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 31.78m ± 0% 14.97m ± 0% -52.88% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 66.14m ± 1% 30.66m ± 0% -53.63% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 107.50m ± 0% 53.44m ± 0% -50.29% (p=0.000 n=10) DiffStack/1_x_33_kB-10 240.51µ ± 1% 76.95µ ± 0% -68.01% (p=0.000 n=10) DiffStack/2_x_33_kB-10 798.5µ ± 0% 310.6µ ± 0% -61.10% (p=0.000 n=10) DiffStack/4_x_33_kB-10 2451.9µ ± 1% 978.9µ ± 3% -60.08% (p=0.000 n=10) DiffStack/8_x_33_kB-10 8.258m ± 0% 3.731m ± 0% -54.82% (p=0.000 n=10) DiffStack/16_x_33_kB-10 28.29m ± 1% 12.77m ± 0% -54.85% (p=0.000 n=10) DiffStack/32_x_33_kB-10 104.11m ± 1% 44.84m ± 1% -56.93% (p=0.000 n=10) DiffStack/48_x_33_kB-10 199.73m ± 1% 95.87m ± 1% -52.00% (p=0.000 n=10) DiffStack/64_x_33_kB-10 339.6m ± 2% 159.2m ± 2% -53.12% (p=0.000 n=10) DiffStack/2_x_131_kB-10 2721.7µ ± 0% 984.3µ ± 0% -63.84% (p=0.000 n=10) DiffStack/4_x_131_kB-10 8.334m ± 1% 3.654m ± 2% -56.15% (p=0.000 n=10) DiffStack/8_x_131_kB-10 25.89m ± 0% 12.23m ± 0% -52.77% (p=0.000 n=10) DiffStack/16_x_131_kB-10 87.49m ± 1% 40.26m ± 0% -53.98% (p=0.000 n=10) DiffStack/32_x_131_kB-10 317.0m ± 5% 150.4m ± 0% -52.57% (p=0.000 n=10) DiffStack/48_x_131_kB-10 681.8m ± 1% 332.1m ± 0% -51.28% (p=0.000 n=10) DiffStack/64_x_131_kB-10 1190.4m ± 1% 583.1m ± 0% -51.01% (p=0.000 n=10) DiffStack/1_x_524_kB-10 3.215m ± 0% 1.086m ± 0% -66.21% (p=0.000 n=10) DiffStack/2_x_524_kB-10 9.676m ± 1% 3.994m ± 0% -58.72% (p=0.000 n=10) DiffStack/4_x_524_kB-10 29.84m ± 0% 12.88m ± 1% -56.84% (p=0.000 n=10) DiffStack/8_x_524_kB-10 92.11m ± 1% 42.35m ± 0% -54.02% (p=0.000 n=10) DiffStack/16_x_524_kB-10 322.1m ± 0% 154.0m ± 0% -52.19% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 54.73m ± 1% 22.63m ± 0% -58.65% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 815.1m ± 0% 395.5m ± 2% -51.48% (p=0.000 n=10) geomean 10.45m 4.530m -56.65% │ buffer-pool.txt │ spanned.txt │ │ checkpoint_bytes │ checkpoint_bytes vs base │ DiffStack/1_x_2_B-10 2.539k ± 0% 1.950k ± 0% -23.20% (p=0.000 n=10) DiffStack/2_x_2_B-10 5.201k ± 0% 4.548k ± 0% -12.56% (p=0.000 n=10) DiffStack/4_x_2_B-10 13.63k ± 0% 12.30k ± 0% -9.77% (p=0.000 n=10) DiffStack/8_x_2_B-10 45.09k ± 0% 42.86k ± 0% -4.95% (p=0.000 n=10) DiffStack/16_x_2_B-10 161.9k ± 0% 156.4k ± 0% -3.42% (p=0.000 n=10) DiffStack/32_x_2_B-10 579.8k ± 0% 604.2k ± 0% +4.21% (p=0.000 n=10) DiffStack/48_x_2_B-10 1.306M ± 0% 1.180M ± 0% -9.68% (p=0.000 n=10) DiffStack/64_x_2_B-10 2.164M ± 0% 2.134M ± 0% -1.36% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 27.109k ± 0% 9.856k ± 0% -63.64% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 78.91k ± 0% 61.59k ± 0% -21.94% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 284.2k ± 0% 258.3k ± 0% -9.13% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 1.119M ± 0% 1.041M ± 0% -6.89% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 4.389M ± 0% 4.201M ± 0% -4.29% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 16.74M ± 0% 17.06M ± 0% +1.95% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 36.30M ± 0% 35.93M ± 0% -1.00% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 60.98M ± 0% 64.06M ± 0% +5.05% (p=0.000 n=10) DiffStack/1_x_33_kB-10 67.79k ± 0% 34.43k ± 0% -49.20% (p=0.000 n=10) DiffStack/2_x_33_kB-10 300.1k ± 0% 233.6k ± 0% -22.15% (p=0.000 n=10) DiffStack/4_x_33_kB-10 1128.3k ± 0% 929.5k ± 0% -17.62% (p=0.000 n=10) DiffStack/8_x_33_kB-10 4.437M ± 0% 4.238M ± 0% -4.49% (p=0.000 n=10) DiffStack/16_x_33_kB-10 16.57M ± 0% 15.78M ± 0% -4.79% (p=0.000 n=10) DiffStack/32_x_33_kB-10 64.35M ± 0% 58.66M ± 0% -8.84% (p=0.000 n=10) DiffStack/48_x_33_kB-10 127.0M ± 0% 128.0M ± 0% +0.80% (p=0.000 n=10) DiffStack/64_x_33_kB-10 217.7M ± 0% 216.5M ± 0% -0.54% (p=0.000 n=10) DiffStack/2_x_131_kB-10 1316.2k ± 0% 921.8k ± 0% -29.97% (p=0.000 n=10) DiffStack/4_x_131_kB-10 4.733M ± 0% 4.207M ± 0% -11.11% (p=0.000 n=10) DiffStack/8_x_131_kB-10 16.03M ± 0% 15.77M ± 0% -1.64% (p=0.000 n=10) DiffStack/16_x_131_kB-10 57.42M ± 0% 54.66M ± 0% -4.81% (p=0.000 n=10) DiffStack/32_x_131_kB-10 214.8M ± 0% 210.7M ± 0% -1.90% (p=0.000 n=10) DiffStack/48_x_131_kB-10 472.9M ± 0% 466.9M ± 0% -1.28% (p=0.000 n=10) DiffStack/64_x_131_kB-10 832.3M ± 0% 824.7M ± 0% -0.92% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1.575M ± 0% 1.051M ± 0% -33.32% (p=0.000 n=10) DiffStack/2_x_524_kB-10 5.248M ± 0% 4.199M ± 0% -20.00% (p=0.000 n=10) DiffStack/4_x_524_kB-10 18.36M ± 0% 15.74M ± 0% -14.28% (p=0.000 n=10) DiffStack/8_x_524_kB-10 60.86M ± 0% 56.66M ± 0% -6.90% (p=0.000 n=10) DiffStack/16_x_524_kB-10 223.5M ± 0% 214.0M ± 0% -4.23% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 13.21M ± 0% 13.20M ± 0% -0.01% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 155.9M ± 0% 154.9M ± 0% -0.60% (p=0.000 n=10) geomean 3.606M 3.173M -12.01% │ buffer-pool.txt │ spanned.txt │ │ ratio │ ratio vs base │ DiffStack/1_x_2_B-10 789.5m ± 0% 1641.0m ± 0% +107.85% (p=0.000 n=10) DiffStack/2_x_2_B-10 1.034 ± 0% 2.353 ± 0% +127.56% (p=0.000 n=10) DiffStack/4_x_2_B-10 1.575 ± 0% 3.592 ± 0% +128.06% (p=0.000 n=10) DiffStack/8_x_2_B-10 2.829 ± 0% 6.684 ± 0% +136.27% (p=0.000 n=10) DiffStack/16_x_2_B-10 5.296 ± 0% 12.610 ± 0% +138.10% (p=0.000 n=10) DiffStack/32_x_2_B-10 9.681 ± 0% 24.790 ± 0% +156.07% (p=0.000 n=10) DiffStack/48_x_2_B-10 14.65 ± 0% 32.44 ± 0% +121.43% (p=0.000 n=10) DiffStack/64_x_2_B-10 18.26 ± 0% 44.11 ± 0% +141.57% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 1.383 ± 0% 1.051 ± 0% -24.01% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 1.716 ± 0% 2.324 ± 0% +35.43% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 2.878 ± 0% 4.251 ± 0% +47.71% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 5.475 ± 0% 8.056 ± 0% +47.14% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.56 ± 0% 15.77 ± 0% +49.34% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 19.97 ± 0% 31.57 ± 0% +58.09% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 28.80 ± 0% 44.11 ± 0% +53.16% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 36.23 ± 0% 58.84 ± 0% +62.41% (p=0.000 n=10) DiffStack/1_x_33_kB-10 986.0m ± 0% 1014.0m ± 0% +2.84% (p=0.000 n=10) DiffStack/2_x_33_kB-10 1.777 ± 0% 2.331 ± 0% +31.18% (p=0.000 n=10) DiffStack/4_x_33_kB-10 3.057 ± 0% 3.993 ± 0% +30.62% (p=0.000 n=10) DiffStack/8_x_33_kB-10 5.766 ± 0% 8.511 ± 0% +47.61% (p=0.000 n=10) DiffStack/16_x_33_kB-10 10.55 ± 0% 15.35 ± 0% +45.50% (p=0.000 n=10) DiffStack/32_x_33_kB-10 20.28 ± 0% 28.08 ± 0% +38.46% (p=0.000 n=10) DiffStack/48_x_33_kB-10 26.59 ± 0% 40.64 ± 0% +52.84% (p=0.000 n=10) DiffStack/64_x_33_kB-10 34.14 ± 0% 51.44 ± 0% +50.67% (p=0.000 n=10) DiffStack/2_x_131_kB-10 1.993 ± 0% 2.333 ± 0% +17.06% (p=0.000 n=10) DiffStack/4_x_131_kB-10 3.263 ± 0% 4.569 ± 0% +40.02% (p=0.000 n=10) DiffStack/8_x_131_kB-10 5.291 ± 0% 7.995 ± 0% +51.11% (p=0.000 n=10) DiffStack/16_x_131_kB-10 9.275 ± 0% 13.410 ± 0% +44.58% (p=0.000 n=10) DiffStack/32_x_131_kB-10 17.17 ± 0% 25.44 ± 0% +48.17% (p=0.000 n=10) DiffStack/48_x_131_kB-10 25.11 ± 0% 37.39 ± 0% +48.90% (p=0.000 n=10) DiffStack/64_x_131_kB-10 33.09 ± 0% 49.40 ± 0% +49.29% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1.498 ± 0% 1.999 ± 0% +33.44% (p=0.000 n=10) DiffStack/2_x_524_kB-10 1.998 ± 0% 2.666 ± 0% +33.43% (p=0.000 n=10) DiffStack/4_x_524_kB-10 3.180 ± 0% 4.285 ± 0% +34.75% (p=0.000 n=10) DiffStack/8_x_524_kB-10 5.040 ± 0% 7.199 ± 0% +42.84% (p=0.000 n=10) DiffStack/16_x_524_kB-10 9.058 ± 0% 13.160 ± 0% +45.29% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 1.997 ± 0% 1.999 ± 0% +0.10% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 36.60 ± 0% 40.42 ± 0% +10.44% (p=0.000 n=10) geomean 5.993 9.122 +52.20% │ buffer-pool.txt │ spanned.txt │ │ wire_bytes │ wire_bytes vs base │ DiffStack/1_x_2_B-10 3.216k ± 0% 1.188k ± 0% -63.06% (p=0.000 n=10) DiffStack/2_x_2_B-10 5.031k ± 0% 1.933k ± 0% -61.58% (p=0.000 n=10) DiffStack/4_x_2_B-10 8.655k ± 0% 3.424k ± 0% -60.44% (p=0.000 n=10) DiffStack/8_x_2_B-10 15.938k ± 0% 6.412k ± 0% -59.77% (p=0.000 n=10) DiffStack/16_x_2_B-10 30.58k ± 0% 12.40k ± 0% -59.45% (p=0.000 n=10) DiffStack/32_x_2_B-10 59.89k ± 0% 24.37k ± 0% -59.31% (p=0.000 n=10) DiffStack/48_x_2_B-10 89.17k ± 0% 36.38k ± 0% -59.21% (p=0.000 n=10) DiffStack/64_x_2_B-10 118.48k ± 0% 48.39k ± 0% -59.16% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 19.596k ± 0% 9.378k ± 0% -52.14% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 45.98k ± 0% 26.50k ± 0% -42.36% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 98.75k ± 0% 60.76k ± 0% -38.47% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 204.3k ± 0% 129.3k ± 0% -36.72% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 415.5k ± 0% 266.3k ± 0% -35.91% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 837.9k ± 0% 540.4k ± 0% -35.50% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 1260.4k ± 0% 814.6k ± 0% -35.37% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 1.683M ± 0% 1.089M ± 0% -35.31% (p=0.000 n=10) DiffStack/1_x_33_kB-10 68.75k ± 0% 33.95k ± 0% -50.61% (p=0.000 n=10) DiffStack/2_x_33_kB-10 168.9k ± 0% 100.2k ± 0% -40.64% (p=0.000 n=10) DiffStack/4_x_33_kB-10 369.1k ± 0% 232.8k ± 0% -36.93% (p=0.000 n=10) DiffStack/8_x_33_kB-10 769.6k ± 0% 497.9k ± 0% -35.30% (p=0.000 n=10) DiffStack/16_x_33_kB-10 1.571M ± 0% 1.028M ± 0% -34.53% (p=0.000 n=10) DiffStack/32_x_33_kB-10 3.173M ± 0% 2.089M ± 0% -34.16% (p=0.000 n=10) DiffStack/48_x_33_kB-10 4.775M ± 0% 3.149M ± 0% -34.04% (p=0.000 n=10) DiffStack/64_x_33_kB-10 6.377M ± 0% 4.210M ± 0% -33.98% (p=0.000 n=10) DiffStack/2_x_131_kB-10 660.4k ± 0% 395.1k ± 0% -40.16% (p=0.000 n=10) DiffStack/4_x_131_kB-10 1450.4k ± 0% 920.9k ± 0% -36.51% (p=0.000 n=10) DiffStack/8_x_131_kB-10 3.031M ± 0% 1.972M ± 0% -34.91% (p=0.000 n=10) DiffStack/16_x_131_kB-10 6.191M ± 0% 4.076M ± 0% -34.17% (p=0.000 n=10) DiffStack/32_x_131_kB-10 12.512M ± 0% 8.282M ± 0% -33.81% (p=0.000 n=10) DiffStack/48_x_131_kB-10 18.83M ± 0% 12.49M ± 0% -33.69% (p=0.000 n=10) DiffStack/64_x_131_kB-10 25.15M ± 0% 16.69M ± 0% -33.63% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1051.8k ± 0% 525.5k ± 0% -50.04% (p=0.000 n=10) DiffStack/2_x_524_kB-10 2.626M ± 0% 1.575M ± 0% -40.04% (p=0.000 n=10) DiffStack/4_x_524_kB-10 5.776M ± 0% 3.673M ± 0% -36.40% (p=0.000 n=10) DiffStack/8_x_524_kB-10 12.075M ± 0% 7.871M ± 0% -34.82% (p=0.000 n=10) DiffStack/16_x_524_kB-10 24.67M ± 0% 16.27M ± 0% -34.07% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 6.612M ± 0% 6.606M ± 0% -0.10% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 4.259M ± 0% 3.833M ± 0% -10.00% (p=0.000 n=10) geomean 601.7k 347.8k -42.19% │ buffer-pool.txt │ spanned.txt │ │ B/op │ B/op vs base │ DiffStack/1_x_2_B-10 31.85Ki ± 0% 15.79Ki ± 0% -50.42% (p=0.000 n=10) DiffStack/2_x_2_B-10 61.81Ki ± 0% 28.01Ki ± 0% -54.68% (p=0.000 n=10) DiffStack/4_x_2_B-10 147.58Ki ± 0% 55.60Ki ± 0% -62.33% (p=0.000 n=10) DiffStack/8_x_2_B-10 425.9Ki ± 0% 116.9Ki ± 0% -72.55% (p=0.000 n=10) DiffStack/16_x_2_B-10 1429.9Ki ± 0% 307.8Ki ± 0% -78.47% (p=0.000 n=10) DiffStack/32_x_2_B-10 4955.7Ki ± 0% 934.1Ki ± 0% -81.15% (p=0.000 n=10) DiffStack/48_x_2_B-10 10.746Mi ± 0% 1.682Mi ± 0% -84.34% (p=0.000 n=10) DiffStack/64_x_2_B-10 17.664Mi ± 0% 2.822Mi ± 0% -84.03% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 202.50Ki ± 0% 41.18Ki ± 0% -79.66% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 511.5Ki ± 0% 134.1Ki ± 0% -73.78% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1513.5Ki ± 0% 309.4Ki ± 0% -79.56% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 4886.0Ki ± 0% 674.2Ki ± 0% -86.20% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 16.287Mi ± 0% 1.427Mi ± 0% -91.24% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 57.541Mi ± 0% 3.198Mi ± 0% -94.44% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 121.703Mi ± 0% 5.810Mi ± 0% -95.23% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 200.588Mi ± 0% 7.495Mi ± 0% -96.26% (p=0.000 n=10) DiffStack/1_x_33_kB-10 562.5Ki ± 0% 136.3Ki ± 0% -75.78% (p=0.000 n=10) DiffStack/2_x_33_kB-10 2041.0Ki ± 0% 505.5Ki ± 0% -75.23% (p=0.000 n=10) DiffStack/4_x_33_kB-10 5.874Mi ± 0% 1.129Mi ± 0% -80.78% (p=0.000 n=10) DiffStack/8_x_33_kB-10 18.837Mi ± 0% 2.410Mi ± 0% -87.21% (p=0.000 n=10) DiffStack/16_x_33_kB-10 61.898Mi ± 0% 5.015Mi ± 0% -91.90% (p=0.000 n=10) DiffStack/32_x_33_kB-10 221.69Mi ± 0% 10.36Mi ± 0% -95.32% (p=0.000 n=10) DiffStack/48_x_33_kB-10 430.13Mi ± 0% 18.50Mi ± 0% -95.70% (p=0.000 n=10) DiffStack/64_x_33_kB-10 722.07Mi ± 0% 21.63Mi ± 0% -97.00% (p=0.000 n=10) DiffStack/2_x_131_kB-10 8.098Mi ± 0% 1.656Mi ± 0% -79.55% (p=0.000 n=10) DiffStack/4_x_131_kB-10 22.662Mi ± 0% 3.934Mi ± 0% -82.64% (p=0.000 n=10) DiffStack/8_x_131_kB-10 65.217Mi ± 0% 8.425Mi ± 0% -87.08% (p=0.000 n=10) DiffStack/16_x_131_kB-10 204.90Mi ± 0% 17.22Mi ± 1% -91.60% (p=0.000 n=10) DiffStack/32_x_131_kB-10 703.12Mi ± 0% 34.89Mi ± 0% -95.04% (p=0.000 n=10) DiffStack/48_x_131_kB-10 1505.36Mi ± 0% 61.24Mi ± 0% -95.93% (p=0.000 n=10) DiffStack/64_x_131_kB-10 2586.04Mi ± 0% 70.48Mi ± 1% -97.27% (p=0.000 n=10) DiffStack/1_x_524_kB-10 11.854Mi ± 1% 2.127Mi ± 0% -82.06% (p=0.000 n=10) DiffStack/2_x_524_kB-10 32.932Mi ± 1% 6.558Mi ± 1% -80.09% (p=0.000 n=10) DiffStack/4_x_524_kB-10 90.04Mi ± 0% 15.15Mi ± 0% -83.17% (p=0.000 n=10) DiffStack/8_x_524_kB-10 252.94Mi ± 0% 32.27Mi ± 1% -87.24% (p=0.000 n=10) DiffStack/16_x_524_kB-10 801.77Mi ± 0% 66.46Mi ± 1% -91.71% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 103.67Mi ± 0% 35.52Mi ± 0% -65.74% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 913.90Mi ± 0% 53.53Mi ± 1% -94.14% (p=0.000 n=10) geomean 19.16Mi 2.409Mi -87.43% │ buffer-pool.txt │ spanned.txt │ │ allocs/op │ allocs/op vs base │ DiffStack/1_x_2_B-10 344.0 ± 0% 184.0 ± 0% -46.51% (p=0.000 n=10) DiffStack/2_x_2_B-10 616.0 ± 0% 295.0 ± 0% -52.11% (p=0.000 n=10) DiffStack/4_x_2_B-10 1381.0 ± 0% 524.0 ± 0% -62.06% (p=0.000 n=10) DiffStack/8_x_2_B-10 3.922k ± 0% 1.032k ± 0% -73.69% (p=0.000 n=10) DiffStack/16_x_2_B-10 12.791k ± 0% 2.233k ± 0% -82.54% (p=0.000 n=10) DiffStack/32_x_2_B-10 43.416k ± 0% 5.410k ± 0% -87.54% (p=0.000 n=10) DiffStack/48_x_2_B-10 95.803k ± 0% 9.036k ± 0% -90.57% (p=0.000 n=10) DiffStack/64_x_2_B-10 156.99k ± 0% 13.99k ± 0% -91.09% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 335.0 ± 0% 182.0 ± 0% -45.67% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 597.5 ± 0% 295.0 ± 0% -50.63% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1304.0 ± 0% 527.0 ± 0% -59.59% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 3.475k ± 0% 1.036k ± 0% -70.19% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.677k ± 0% 2.239k ± 0% -79.03% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 35.357k ± 0% 5.414k ± 0% -84.69% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 72.782k ± 0% 9.296k ± 0% -87.23% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 119.24k ± 0% 14.26k ± 0% -88.04% (p=0.000 n=10) DiffStack/1_x_33_kB-10 309.0 ± 0% 182.0 ± 0% -41.10% (p=0.000 n=10) DiffStack/2_x_33_kB-10 594.0 ± 0% 296.0 ± 0% -50.17% (p=0.000 n=10) DiffStack/4_x_33_kB-10 1324.5 ± 0% 528.0 ± 0% -60.14% (p=0.000 n=10) DiffStack/8_x_33_kB-10 3.554k ± 0% 1.047k ± 0% -70.54% (p=0.000 n=10) DiffStack/16_x_33_kB-10 10.511k ± 0% 2.232k ± 0% -78.77% (p=0.000 n=10) DiffStack/32_x_33_kB-10 35.147k ± 0% 5.194k ± 0% -85.22% (p=0.000 n=10) DiffStack/48_x_33_kB-10 66.519k ± 0% 8.958k ± 0% -86.53% (p=0.000 n=10) DiffStack/64_x_33_kB-10 110.71k ± 0% 13.28k ± 0% -88.00% (p=0.000 n=10) DiffStack/2_x_131_kB-10 644.0 ± 0% 296.0 ± 0% -54.04% (p=0.000 n=10) DiffStack/4_x_131_kB-10 1418.5 ± 0% 535.0 ± 0% -62.28% (p=0.000 n=10) DiffStack/8_x_131_kB-10 3.497k ± 0% 1.047k ± 0% -70.06% (p=0.000 n=10) DiffStack/16_x_131_kB-10 9.805k ± 0% 2.182k ± 0% -77.75% (p=0.000 n=10) DiffStack/32_x_131_kB-10 30.996k ± 0% 5.040k ± 0% -83.74% (p=0.000 n=10) DiffStack/48_x_131_kB-10 63.751k ± 0% 8.656k ± 0% -86.42% (p=0.000 n=10) DiffStack/64_x_131_kB-10 107.97k ± 0% 13.03k ± 0% -87.93% (p=0.000 n=10) DiffStack/1_x_524_kB-10 356.0 ± 0% 186.0 ± 0% -47.75% (p=0.000 n=10) DiffStack/2_x_524_kB-10 671.0 ± 0% 300.0 ± 0% -55.29% (p=0.000 n=10) DiffStack/4_x_524_kB-10 1467.5 ± 0% 536.0 ± 0% -63.48% (p=0.000 n=10) DiffStack/8_x_524_kB-10 3.553k ± 0% 1.038k ± 0% -70.80% (p=0.000 n=10) DiffStack/16_x_524_kB-10 9.971k ± 0% 2.182k ± 0% -78.12% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 489.16k ± 0% 73.65k ± 0% -84.94% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 7236.9k ± 0% 857.8k ± 0% -88.15% (p=0.000 n=10) geomean 7.989k 1.934k -75.79% ```
2023-05-05 02:15:09 +00:00
//go:build !go1.20
// Copyright 2016-2022, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package client
import (
"bytes"
"encoding/json"
"io"
"math"
[snapshot] Pool checkpoint buffers Pool the buffers used for marshaling checkpoints. The pooling works by returning the last saved checkpoint's buffer to the diff state when a new checkpoint is saved. If a pooled buffer is available, it is reused when a checkpoint is marshaled. With our current usage patterns, this will cycle through two buffers. Starting from a fresh diff state: - The first call to MarshalDeployment will create a new buffer B1 - The first call to Saved will put B1 into lastSavedDeployment - The second call to MarshalDeployment will create a new buffer B2 - The second call to Saved will put B1 into the pool and B2 into lastSavedDeployment - The third call to MarshalDeployment will get B1 from the pool - The third call to Saved will put B2 into the pool and B1 into lastSavedDeployment These changes reduce CPU time and allocation volume by a geomean of 6% and 27%, respectively. Benchmark results and analysis: httpstate ❯ go test -count=10 -run none -benchmem -bench . | tee buffer-pool.txt httpstate ❯ benchstat base.txt buffer-pool.txt >stat.txt ``` │ base.txt │ buffer-pool.txt │ │ sec/op │ sec/op vs base │ DiffStack/1_x_2_B-10 46.30µ ± 1% 43.17µ ± 0% -6.74% (p=0.000 n=10) DiffStack/2_x_2_B-10 82.68µ ± 1% 80.07µ ± 0% -3.16% (p=0.000 n=10) DiffStack/4_x_2_B-10 191.0µ ± 1% 175.3µ ± 1% -8.22% (p=0.000 n=10) DiffStack/8_x_2_B-10 499.7µ ± 0% 470.6µ ± 1% -5.83% (p=0.000 n=10) DiffStack/16_x_2_B-10 1.489m ± 0% 1.400m ± 1% -5.92% (p=0.000 n=10) DiffStack/32_x_2_B-10 4.895m ± 1% 4.540m ± 0% -7.25% (p=0.000 n=10) DiffStack/48_x_2_B-10 9.651m ± 1% 8.538m ± 1% -11.53% (p=0.000 n=10) DiffStack/64_x_2_B-10 15.11m ± 0% 13.82m ± 0% -8.56% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 103.3µ ± 1% 123.9µ ± 1% +19.93% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 308.8µ ± 0% 317.1µ ± 1% +2.70% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 899.2µ ± 0% 835.6µ ± 0% -7.07% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 2.989m ± 1% 2.757m ± 1% -7.76% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.528m ± 0% 9.305m ± 1% -11.61% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 36.27m ± 1% 30.55m ± 1% -15.77% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 72.53m ± 0% 69.05m ± 0% -4.80% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 125.5m ± 0% 113.5m ± 0% -9.58% (p=0.000 n=10) DiffStack/1_x_33_kB-10 317.4µ ± 1% 314.2µ ± 0% -1.01% (p=0.001 n=10) DiffStack/2_x_33_kB-10 756.8µ ± 1% 797.3µ ± 1% +5.35% (p=0.000 n=10) DiffStack/4_x_33_kB-10 2.605m ± 0% 2.371m ± 0% -8.99% (p=0.000 n=10) DiffStack/8_x_33_kB-10 9.241m ± 2% 8.342m ± 0% -9.73% (p=0.000 n=10) DiffStack/16_x_33_kB-10 32.86m ± 1% 27.11m ± 1% -17.50% (p=0.000 n=10) DiffStack/32_x_33_kB-10 110.2m ± 2% 101.3m ± 1% -8.01% (p=0.000 n=10) DiffStack/48_x_33_kB-10 231.3m ± 0% 202.1m ± 1% -12.64% (p=0.000 n=10) DiffStack/64_x_33_kB-10 383.8m ± 1% 333.4m ± 1% -13.14% (p=0.000 n=10) DiffStack/2_x_131_kB-10 2.748m ± 1% 2.678m ± 0% -2.52% (p=0.000 n=10) DiffStack/4_x_131_kB-10 8.609m ± 0% 8.278m ± 0% -3.85% (p=0.000 n=10) DiffStack/8_x_131_kB-10 28.44m ± 0% 25.46m ± 0% -10.49% (p=0.000 n=10) DiffStack/16_x_131_kB-10 96.98m ± 1% 88.22m ± 0% -9.03% (p=0.000 n=10) DiffStack/32_x_131_kB-10 349.4m ± 1% 315.3m ± 0% -9.74% (p=0.000 n=10) DiffStack/48_x_131_kB-10 761.7m ± 0% 677.9m ± 1% -11.00% (p=0.000 n=10) DiffStack/64_x_131_kB-10 1.312 ± 1% 1.195 ± 0% -8.90% (p=0.000 n=10) DiffStack/1_x_524_kB-10 3.238m ± 1% 3.213m ± 0% -0.78% (p=0.000 n=10) DiffStack/2_x_524_kB-10 9.867m ± 0% 9.613m ± 0% -2.58% (p=0.000 n=10) DiffStack/4_x_524_kB-10 29.65m ± 0% 29.11m ± 0% -1.83% (p=0.000 n=10) DiffStack/8_x_524_kB-10 98.91m ± 1% 92.23m ± 0% -6.76% (p=0.000 n=10) DiffStack/16_x_524_kB-10 348.1m ± 1% 322.5m ± 1% -7.34% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 46.62m ± 0% 46.42m ± 1% -0.42% (p=0.004 n=10) DiffStackRecorded/checkpoints.json-10 835.8m ± 0% 809.5m ± 0% -3.15% (p=0.000 n=10) geomean 11.15m 10.44m -6.39% │ base.txt │ buffer-pool.txt │ │ ratio │ ratio vs base │ DiffStack/1_x_2_B-10 789.5m ± 0% 700.9m ± 0% -11.22% (p=0.000 n=10) DiffStack/2_x_2_B-10 1.034 ± 0% 1.030 ± 0% -0.39% (p=0.000 n=10) DiffStack/4_x_2_B-10 1.676 ± 0% 1.608 ± 0% -4.06% (p=0.000 n=10) DiffStack/8_x_2_B-10 2.883 ± 0% 2.879 ± 0% -0.14% (p=0.000 n=10) DiffStack/16_x_2_B-10 5.270 ± 0% 5.268 ± 0% -0.04% (p=0.000 n=10) DiffStack/32_x_2_B-10 10.20 ± 0% 10.03 ± 0% -1.67% (p=0.000 n=10) DiffStack/48_x_2_B-10 14.24 ± 0% 13.39 ± 0% -5.97% (p=0.000 n=10) DiffStack/64_x_2_B-10 17.51 ± 0% 17.34 ± 0% -0.97% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 950.9m ± 0% 1383.0m ± 0% +45.44% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 1.716 ± 0% 1.900 ± 0% +10.72% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 2.792 ± 0% 2.792 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/8_x_8.2_kB-10 5.350 ± 0% 5.516 ± 0% +3.10% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.62 ± 0% 10.60 ± 0% -0.19% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 20.01 ± 0% 19.10 ± 0% -4.55% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 27.48 ± 0% 30.24 ± 0% +10.04% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 36.97 ± 0% 38.22 ± 0% +3.38% (p=0.000 n=10) DiffStack/1_x_33_kB-10 1.467 ± 0% 1.467 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/2_x_33_kB-10 1.581 ± 0% 1.777 ± 0% +12.40% (p=0.000 n=10) DiffStack/4_x_33_kB-10 2.967 ± 0% 2.967 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/8_x_33_kB-10 5.766 ± 0% 5.809 ± 0% +0.75% (p=0.000 n=10) DiffStack/16_x_33_kB-10 10.87 ± 0% 10.05 ± 0% -7.54% (p=0.000 n=10) DiffStack/32_x_33_kB-10 19.00 ± 0% 20.13 ± 0% +5.95% (p=0.000 n=10) DiffStack/48_x_33_kB-10 27.14 ± 0% 27.18 ± 0% +0.15% (p=0.000 n=10) DiffStack/64_x_33_kB-10 34.39 ± 0% 33.97 ± 0% -1.22% (p=0.000 n=10) DiffStack/2_x_131_kB-10 1.993 ± 0% 1.993 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/4_x_131_kB-10 3.173 ± 0% 3.263 ± 0% +2.84% (p=0.000 n=10) DiffStack/8_x_131_kB-10 5.334 ± 0% 5.291 ± 0% -0.81% (p=0.000 n=10) DiffStack/16_x_131_kB-10 9.296 ± 0% 9.423 ± 0% +1.37% (p=0.000 n=10) DiffStack/32_x_131_kB-10 17.10 ± 0% 17.21 ± 0% +0.64% (p=0.000 n=10) DiffStack/48_x_131_kB-10 25.25 ± 0% 25.06 ± 0% -0.75% (p=0.000 n=10) DiffStack/64_x_131_kB-10 33.13 ± 0% 33.46 ± 0% +1.00% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1.498 ± 0% 1.498 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/2_x_524_kB-10 1.998 ± 0% 1.998 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/4_x_524_kB-10 2.998 ± 0% 3.089 ± 0% +3.04% (p=0.000 n=10) DiffStack/8_x_524_kB-10 5.084 ± 0% 5.084 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/16_x_524_kB-10 9.079 ± 0% 9.058 ± 0% -0.23% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 1.997 ± 0% 1.997 ± 0% ~ (p=1.000 n=10) ¹ DiffStackRecorded/checkpoints.json-10 36.60 ± 0% 36.60 ± 0% ~ (p=1.000 n=10) ¹ geomean 5.961 6.039 +1.31% ¹ all samples are equal │ base.txt │ buffer-pool.txt │ │ B/op │ B/op vs base │ DiffStack/1_x_2_B-10 35.17Ki ± 0% 29.56Ki ± 0% -15.97% (p=0.000 n=10) DiffStack/2_x_2_B-10 70.05Ki ± 0% 61.61Ki ± 0% -12.04% (p=0.000 n=10) DiffStack/4_x_2_B-10 186.8Ki ± 0% 149.9Ki ± 0% -19.75% (p=0.000 n=10) DiffStack/8_x_2_B-10 529.0Ki ± 0% 432.0Ki ± 0% -18.33% (p=0.000 n=10) DiffStack/16_x_2_B-10 1.768Mi ± 0% 1.392Mi ± 0% -21.28% (p=0.000 n=10) DiffStack/32_x_2_B-10 6.513Mi ± 0% 4.986Mi ± 0% -23.45% (p=0.000 n=10) DiffStack/48_x_2_B-10 14.054Mi ± 0% 9.885Mi ± 0% -29.66% (p=0.000 n=10) DiffStack/64_x_2_B-10 22.39Mi ± 0% 16.84Mi ± 0% -24.79% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 154.7Ki ± 0% 202.4Ki ± 0% +30.89% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 571.9Ki ± 0% 540.8Ki ± 0% -5.43% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1.846Mi ± 0% 1.430Mi ± 0% -22.52% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 6.920Mi ± 0% 4.809Mi ± 0% -30.51% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 26.50Mi ± 0% 16.34Mi ± 0% -38.35% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 98.33Mi ± 0% 55.24Mi ± 0% -43.82% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 209.4Mi ± 0% 127.2Mi ± 0% -39.23% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 365.6Mi ± 0% 211.2Mi ± 0% -42.23% (p=0.000 n=10) DiffStack/1_x_33_kB-10 808.7Ki ± 0% 764.2Ki ± 0% -5.50% (p=0.000 n=10) DiffStack/2_x_33_kB-10 2.013Mi ± 0% 1.992Mi ± 0% -1.03% (p=0.000 n=10) DiffStack/4_x_33_kB-10 7.551Mi ± 0% 5.764Mi ± 0% -23.67% (p=0.000 n=10) DiffStack/8_x_33_kB-10 28.58Mi ± 0% 18.90Mi ± 0% -33.87% (p=0.000 n=10) DiffStack/16_x_33_kB-10 104.40Mi ± 0% 59.42Mi ± 0% -43.08% (p=0.000 n=10) DiffStack/32_x_33_kB-10 365.5Mi ± 0% 219.9Mi ± 0% -39.82% (p=0.000 n=10) DiffStack/48_x_33_kB-10 801.1Mi ± 0% 438.7Mi ± 0% -45.24% (p=0.000 n=10) DiffStack/64_x_33_kB-10 1314.3Mi ± 0% 718.3Mi ± 0% -45.35% (p=0.000 n=10) DiffStack/2_x_131_kB-10 9.179Mi ± 0% 8.063Mi ± 0% -12.16% (p=0.000 n=10) DiffStack/4_x_131_kB-10 29.14Mi ± 0% 22.68Mi ± 0% -22.18% (p=0.000 n=10) DiffStack/8_x_131_kB-10 98.77Mi ± 0% 65.16Mi ± 0% -34.03% (p=0.000 n=10) DiffStack/16_x_131_kB-10 337.1Mi ± 0% 207.8Mi ± 0% -38.36% (p=0.000 n=10) DiffStack/32_x_131_kB-10 1234.6Mi ± 0% 705.3Mi ± 0% -42.87% (p=0.000 n=10) DiffStack/48_x_131_kB-10 2.771Gi ± 0% 1.468Gi ± 0% -47.03% (p=0.000 n=10) DiffStack/64_x_131_kB-10 4.606Gi ± 0% 2.553Gi ± 0% -44.56% (p=0.000 n=10) DiffStack/1_x_524_kB-10 12.41Mi ± 1% 11.85Mi ± 1% -4.50% (p=0.000 n=10) DiffStack/2_x_524_kB-10 37.01Mi ± 1% 32.79Mi ± 1% -11.41% (p=0.000 n=10) DiffStack/4_x_524_kB-10 110.56Mi ± 0% 89.08Mi ± 1% -19.43% (p=0.000 n=10) DiffStack/8_x_524_kB-10 367.3Mi ± 1% 254.3Mi ± 1% -30.76% (p=0.000 n=10) DiffStack/16_x_524_kB-10 1291.7Mi ± 0% 801.6Mi ± 0% -37.95% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 69.25Mi ± 0% 69.25Mi ± 0% ~ (p=0.165 n=10) DiffStackRecorded/checkpoints.json-10 1226.7Mi ± 0% 914.2Mi ± 0% -25.47% (p=0.000 n=10) geomean 26.14Mi 19.06Mi -27.07% │ base.txt │ buffer-pool.txt │ │ allocs/op │ allocs/op vs base │ DiffStack/1_x_2_B-10 361.0 ± 0% 326.0 ± 0% -9.70% (p=0.000 n=10) DiffStack/2_x_2_B-10 649.0 ± 0% 621.0 ± 0% -4.31% (p=0.000 n=10) DiffStack/4_x_2_B-10 1.512k ± 0% 1.399k ± 0% -7.47% (p=0.000 n=10) DiffStack/8_x_2_B-10 4.141k ± 0% 4.006k ± 0% -3.25% (p=0.000 n=10) DiffStack/16_x_2_B-10 13.09k ± 0% 12.73k ± 0% -2.72% (p=0.000 n=10) DiffStack/32_x_2_B-10 46.38k ± 0% 44.88k ± 0% -3.25% (p=0.000 n=10) DiffStack/48_x_2_B-10 94.38k ± 0% 87.94k ± 0% -6.82% (p=0.000 n=10) DiffStack/64_x_2_B-10 152.7k ± 0% 149.3k ± 0% -2.18% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 323.0 ± 0% 335.0 ± 0% +3.72% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 630.0 ± 0% 613.0 ± 0% -2.70% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1.354k ± 0% 1.279k ± 0% -5.54% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 3.606k ± 0% 3.490k ± 0% -3.22% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 11.16k ± 0% 10.71k ± 0% -4.11% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 36.45k ± 0% 34.06k ± 0% -6.54% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 71.55k ± 0% 76.00k ± 0% +6.22% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 123.9k ± 0% 125.2k ± 0% +1.07% (p=0.000 n=10) DiffStack/1_x_33_kB-10 346.0 ± 0% 329.0 ± 0% -4.91% (p=0.000 n=10) DiffStack/2_x_33_kB-10 607.0 ± 0% 594.0 ± 0% -2.14% (p=0.000 n=10) DiffStack/4_x_33_kB-10 1.396k ± 0% 1.310k ± 0% -6.16% (p=0.000 n=10) DiffStack/8_x_33_kB-10 3.764k ± 0% 3.564k ± 0% -5.30% (p=0.000 n=10) DiffStack/16_x_33_kB-10 11.26k ± 0% 10.15k ± 0% -9.88% (p=0.000 n=10) DiffStack/32_x_33_kB-10 34.55k ± 0% 34.91k ± 0% +1.03% (p=0.000 n=10) DiffStack/48_x_33_kB-10 69.72k ± 0% 67.76k ± 0% -2.82% (p=0.000 n=10) DiffStack/64_x_33_kB-10 114.1k ± 0% 110.1k ± 0% -3.46% (p=0.000 n=10) DiffStack/2_x_131_kB-10 678.0 ± 0% 643.0 ± 0% -5.16% (p=0.000 n=10) DiffStack/4_x_131_kB-10 1.494k ± 0% 1.419k ± 0% -4.99% (p=0.000 n=10) DiffStack/8_x_131_kB-10 3.737k ± 0% 3.489k ± 0% -6.64% (p=0.000 n=10) DiffStack/16_x_131_kB-10 10.400k ± 0% 9.917k ± 0% -4.64% (p=0.000 n=10) DiffStack/32_x_131_kB-10 32.24k ± 0% 31.08k ± 0% -3.59% (p=0.000 n=10) DiffStack/48_x_131_kB-10 66.12k ± 0% 63.62k ± 0% -3.79% (p=0.000 n=10) DiffStack/64_x_131_kB-10 110.9k ± 0% 109.1k ± 0% -1.67% (p=0.000 n=10) DiffStack/1_x_524_kB-10 374.0 ± 1% 356.0 ± 0% -4.81% (p=0.000 n=10) DiffStack/2_x_524_kB-10 707.0 ± 0% 668.0 ± 0% -5.52% (p=0.000 n=10) DiffStack/4_x_524_kB-10 1.528k ± 0% 1.460k ± 1% -4.48% (p=0.000 n=10) DiffStack/8_x_524_kB-10 3.797k ± 1% 3.571k ± 0% -5.95% (p=0.000 n=10) DiffStack/16_x_524_kB-10 10.562k ± 1% 9.966k ± 0% -5.65% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 512.1k ± 0% 512.1k ± 0% ~ (p=0.222 n=10) DiffStackRecorded/checkpoints.json-10 7.240M ± 0% 7.237M ± 0% -0.03% (p=0.000 n=10) geomean 8.309k 7.996k -3.77% ```
2023-05-05 00:12:34 +00:00
jsoniter "github.com/json-iterator/go"
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
)
2022-12-15 17:12:32 +00:00
const maxNewLines = 1024
var jsonIterConfig = jsoniter.Config{SortMapKeys: true}.Froze()
// Marshals to canonical JSON in the apitype.UntypedDeployment format.
//
// Optimized for large checkpoints.
//
// Injects newlines to allow efficient textual diffs over the JSON. Textual diffs currently use O(N^2) memory in the
2022-12-15 17:12:32 +00:00
// number of newlines, so the injection needs to be conservative. Currently it limits to up to maxNewLines newlines
// which would result in max 8MB memory use by the algorithm.
[snapshot] Pool checkpoint buffers Pool the buffers used for marshaling checkpoints. The pooling works by returning the last saved checkpoint's buffer to the diff state when a new checkpoint is saved. If a pooled buffer is available, it is reused when a checkpoint is marshaled. With our current usage patterns, this will cycle through two buffers. Starting from a fresh diff state: - The first call to MarshalDeployment will create a new buffer B1 - The first call to Saved will put B1 into lastSavedDeployment - The second call to MarshalDeployment will create a new buffer B2 - The second call to Saved will put B1 into the pool and B2 into lastSavedDeployment - The third call to MarshalDeployment will get B1 from the pool - The third call to Saved will put B2 into the pool and B1 into lastSavedDeployment These changes reduce CPU time and allocation volume by a geomean of 6% and 27%, respectively. Benchmark results and analysis: httpstate ❯ go test -count=10 -run none -benchmem -bench . | tee buffer-pool.txt httpstate ❯ benchstat base.txt buffer-pool.txt >stat.txt ``` │ base.txt │ buffer-pool.txt │ │ sec/op │ sec/op vs base │ DiffStack/1_x_2_B-10 46.30µ ± 1% 43.17µ ± 0% -6.74% (p=0.000 n=10) DiffStack/2_x_2_B-10 82.68µ ± 1% 80.07µ ± 0% -3.16% (p=0.000 n=10) DiffStack/4_x_2_B-10 191.0µ ± 1% 175.3µ ± 1% -8.22% (p=0.000 n=10) DiffStack/8_x_2_B-10 499.7µ ± 0% 470.6µ ± 1% -5.83% (p=0.000 n=10) DiffStack/16_x_2_B-10 1.489m ± 0% 1.400m ± 1% -5.92% (p=0.000 n=10) DiffStack/32_x_2_B-10 4.895m ± 1% 4.540m ± 0% -7.25% (p=0.000 n=10) DiffStack/48_x_2_B-10 9.651m ± 1% 8.538m ± 1% -11.53% (p=0.000 n=10) DiffStack/64_x_2_B-10 15.11m ± 0% 13.82m ± 0% -8.56% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 103.3µ ± 1% 123.9µ ± 1% +19.93% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 308.8µ ± 0% 317.1µ ± 1% +2.70% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 899.2µ ± 0% 835.6µ ± 0% -7.07% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 2.989m ± 1% 2.757m ± 1% -7.76% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.528m ± 0% 9.305m ± 1% -11.61% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 36.27m ± 1% 30.55m ± 1% -15.77% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 72.53m ± 0% 69.05m ± 0% -4.80% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 125.5m ± 0% 113.5m ± 0% -9.58% (p=0.000 n=10) DiffStack/1_x_33_kB-10 317.4µ ± 1% 314.2µ ± 0% -1.01% (p=0.001 n=10) DiffStack/2_x_33_kB-10 756.8µ ± 1% 797.3µ ± 1% +5.35% (p=0.000 n=10) DiffStack/4_x_33_kB-10 2.605m ± 0% 2.371m ± 0% -8.99% (p=0.000 n=10) DiffStack/8_x_33_kB-10 9.241m ± 2% 8.342m ± 0% -9.73% (p=0.000 n=10) DiffStack/16_x_33_kB-10 32.86m ± 1% 27.11m ± 1% -17.50% (p=0.000 n=10) DiffStack/32_x_33_kB-10 110.2m ± 2% 101.3m ± 1% -8.01% (p=0.000 n=10) DiffStack/48_x_33_kB-10 231.3m ± 0% 202.1m ± 1% -12.64% (p=0.000 n=10) DiffStack/64_x_33_kB-10 383.8m ± 1% 333.4m ± 1% -13.14% (p=0.000 n=10) DiffStack/2_x_131_kB-10 2.748m ± 1% 2.678m ± 0% -2.52% (p=0.000 n=10) DiffStack/4_x_131_kB-10 8.609m ± 0% 8.278m ± 0% -3.85% (p=0.000 n=10) DiffStack/8_x_131_kB-10 28.44m ± 0% 25.46m ± 0% -10.49% (p=0.000 n=10) DiffStack/16_x_131_kB-10 96.98m ± 1% 88.22m ± 0% -9.03% (p=0.000 n=10) DiffStack/32_x_131_kB-10 349.4m ± 1% 315.3m ± 0% -9.74% (p=0.000 n=10) DiffStack/48_x_131_kB-10 761.7m ± 0% 677.9m ± 1% -11.00% (p=0.000 n=10) DiffStack/64_x_131_kB-10 1.312 ± 1% 1.195 ± 0% -8.90% (p=0.000 n=10) DiffStack/1_x_524_kB-10 3.238m ± 1% 3.213m ± 0% -0.78% (p=0.000 n=10) DiffStack/2_x_524_kB-10 9.867m ± 0% 9.613m ± 0% -2.58% (p=0.000 n=10) DiffStack/4_x_524_kB-10 29.65m ± 0% 29.11m ± 0% -1.83% (p=0.000 n=10) DiffStack/8_x_524_kB-10 98.91m ± 1% 92.23m ± 0% -6.76% (p=0.000 n=10) DiffStack/16_x_524_kB-10 348.1m ± 1% 322.5m ± 1% -7.34% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 46.62m ± 0% 46.42m ± 1% -0.42% (p=0.004 n=10) DiffStackRecorded/checkpoints.json-10 835.8m ± 0% 809.5m ± 0% -3.15% (p=0.000 n=10) geomean 11.15m 10.44m -6.39% │ base.txt │ buffer-pool.txt │ │ ratio │ ratio vs base │ DiffStack/1_x_2_B-10 789.5m ± 0% 700.9m ± 0% -11.22% (p=0.000 n=10) DiffStack/2_x_2_B-10 1.034 ± 0% 1.030 ± 0% -0.39% (p=0.000 n=10) DiffStack/4_x_2_B-10 1.676 ± 0% 1.608 ± 0% -4.06% (p=0.000 n=10) DiffStack/8_x_2_B-10 2.883 ± 0% 2.879 ± 0% -0.14% (p=0.000 n=10) DiffStack/16_x_2_B-10 5.270 ± 0% 5.268 ± 0% -0.04% (p=0.000 n=10) DiffStack/32_x_2_B-10 10.20 ± 0% 10.03 ± 0% -1.67% (p=0.000 n=10) DiffStack/48_x_2_B-10 14.24 ± 0% 13.39 ± 0% -5.97% (p=0.000 n=10) DiffStack/64_x_2_B-10 17.51 ± 0% 17.34 ± 0% -0.97% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 950.9m ± 0% 1383.0m ± 0% +45.44% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 1.716 ± 0% 1.900 ± 0% +10.72% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 2.792 ± 0% 2.792 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/8_x_8.2_kB-10 5.350 ± 0% 5.516 ± 0% +3.10% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.62 ± 0% 10.60 ± 0% -0.19% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 20.01 ± 0% 19.10 ± 0% -4.55% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 27.48 ± 0% 30.24 ± 0% +10.04% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 36.97 ± 0% 38.22 ± 0% +3.38% (p=0.000 n=10) DiffStack/1_x_33_kB-10 1.467 ± 0% 1.467 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/2_x_33_kB-10 1.581 ± 0% 1.777 ± 0% +12.40% (p=0.000 n=10) DiffStack/4_x_33_kB-10 2.967 ± 0% 2.967 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/8_x_33_kB-10 5.766 ± 0% 5.809 ± 0% +0.75% (p=0.000 n=10) DiffStack/16_x_33_kB-10 10.87 ± 0% 10.05 ± 0% -7.54% (p=0.000 n=10) DiffStack/32_x_33_kB-10 19.00 ± 0% 20.13 ± 0% +5.95% (p=0.000 n=10) DiffStack/48_x_33_kB-10 27.14 ± 0% 27.18 ± 0% +0.15% (p=0.000 n=10) DiffStack/64_x_33_kB-10 34.39 ± 0% 33.97 ± 0% -1.22% (p=0.000 n=10) DiffStack/2_x_131_kB-10 1.993 ± 0% 1.993 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/4_x_131_kB-10 3.173 ± 0% 3.263 ± 0% +2.84% (p=0.000 n=10) DiffStack/8_x_131_kB-10 5.334 ± 0% 5.291 ± 0% -0.81% (p=0.000 n=10) DiffStack/16_x_131_kB-10 9.296 ± 0% 9.423 ± 0% +1.37% (p=0.000 n=10) DiffStack/32_x_131_kB-10 17.10 ± 0% 17.21 ± 0% +0.64% (p=0.000 n=10) DiffStack/48_x_131_kB-10 25.25 ± 0% 25.06 ± 0% -0.75% (p=0.000 n=10) DiffStack/64_x_131_kB-10 33.13 ± 0% 33.46 ± 0% +1.00% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1.498 ± 0% 1.498 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/2_x_524_kB-10 1.998 ± 0% 1.998 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/4_x_524_kB-10 2.998 ± 0% 3.089 ± 0% +3.04% (p=0.000 n=10) DiffStack/8_x_524_kB-10 5.084 ± 0% 5.084 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/16_x_524_kB-10 9.079 ± 0% 9.058 ± 0% -0.23% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 1.997 ± 0% 1.997 ± 0% ~ (p=1.000 n=10) ¹ DiffStackRecorded/checkpoints.json-10 36.60 ± 0% 36.60 ± 0% ~ (p=1.000 n=10) ¹ geomean 5.961 6.039 +1.31% ¹ all samples are equal │ base.txt │ buffer-pool.txt │ │ B/op │ B/op vs base │ DiffStack/1_x_2_B-10 35.17Ki ± 0% 29.56Ki ± 0% -15.97% (p=0.000 n=10) DiffStack/2_x_2_B-10 70.05Ki ± 0% 61.61Ki ± 0% -12.04% (p=0.000 n=10) DiffStack/4_x_2_B-10 186.8Ki ± 0% 149.9Ki ± 0% -19.75% (p=0.000 n=10) DiffStack/8_x_2_B-10 529.0Ki ± 0% 432.0Ki ± 0% -18.33% (p=0.000 n=10) DiffStack/16_x_2_B-10 1.768Mi ± 0% 1.392Mi ± 0% -21.28% (p=0.000 n=10) DiffStack/32_x_2_B-10 6.513Mi ± 0% 4.986Mi ± 0% -23.45% (p=0.000 n=10) DiffStack/48_x_2_B-10 14.054Mi ± 0% 9.885Mi ± 0% -29.66% (p=0.000 n=10) DiffStack/64_x_2_B-10 22.39Mi ± 0% 16.84Mi ± 0% -24.79% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 154.7Ki ± 0% 202.4Ki ± 0% +30.89% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 571.9Ki ± 0% 540.8Ki ± 0% -5.43% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1.846Mi ± 0% 1.430Mi ± 0% -22.52% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 6.920Mi ± 0% 4.809Mi ± 0% -30.51% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 26.50Mi ± 0% 16.34Mi ± 0% -38.35% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 98.33Mi ± 0% 55.24Mi ± 0% -43.82% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 209.4Mi ± 0% 127.2Mi ± 0% -39.23% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 365.6Mi ± 0% 211.2Mi ± 0% -42.23% (p=0.000 n=10) DiffStack/1_x_33_kB-10 808.7Ki ± 0% 764.2Ki ± 0% -5.50% (p=0.000 n=10) DiffStack/2_x_33_kB-10 2.013Mi ± 0% 1.992Mi ± 0% -1.03% (p=0.000 n=10) DiffStack/4_x_33_kB-10 7.551Mi ± 0% 5.764Mi ± 0% -23.67% (p=0.000 n=10) DiffStack/8_x_33_kB-10 28.58Mi ± 0% 18.90Mi ± 0% -33.87% (p=0.000 n=10) DiffStack/16_x_33_kB-10 104.40Mi ± 0% 59.42Mi ± 0% -43.08% (p=0.000 n=10) DiffStack/32_x_33_kB-10 365.5Mi ± 0% 219.9Mi ± 0% -39.82% (p=0.000 n=10) DiffStack/48_x_33_kB-10 801.1Mi ± 0% 438.7Mi ± 0% -45.24% (p=0.000 n=10) DiffStack/64_x_33_kB-10 1314.3Mi ± 0% 718.3Mi ± 0% -45.35% (p=0.000 n=10) DiffStack/2_x_131_kB-10 9.179Mi ± 0% 8.063Mi ± 0% -12.16% (p=0.000 n=10) DiffStack/4_x_131_kB-10 29.14Mi ± 0% 22.68Mi ± 0% -22.18% (p=0.000 n=10) DiffStack/8_x_131_kB-10 98.77Mi ± 0% 65.16Mi ± 0% -34.03% (p=0.000 n=10) DiffStack/16_x_131_kB-10 337.1Mi ± 0% 207.8Mi ± 0% -38.36% (p=0.000 n=10) DiffStack/32_x_131_kB-10 1234.6Mi ± 0% 705.3Mi ± 0% -42.87% (p=0.000 n=10) DiffStack/48_x_131_kB-10 2.771Gi ± 0% 1.468Gi ± 0% -47.03% (p=0.000 n=10) DiffStack/64_x_131_kB-10 4.606Gi ± 0% 2.553Gi ± 0% -44.56% (p=0.000 n=10) DiffStack/1_x_524_kB-10 12.41Mi ± 1% 11.85Mi ± 1% -4.50% (p=0.000 n=10) DiffStack/2_x_524_kB-10 37.01Mi ± 1% 32.79Mi ± 1% -11.41% (p=0.000 n=10) DiffStack/4_x_524_kB-10 110.56Mi ± 0% 89.08Mi ± 1% -19.43% (p=0.000 n=10) DiffStack/8_x_524_kB-10 367.3Mi ± 1% 254.3Mi ± 1% -30.76% (p=0.000 n=10) DiffStack/16_x_524_kB-10 1291.7Mi ± 0% 801.6Mi ± 0% -37.95% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 69.25Mi ± 0% 69.25Mi ± 0% ~ (p=0.165 n=10) DiffStackRecorded/checkpoints.json-10 1226.7Mi ± 0% 914.2Mi ± 0% -25.47% (p=0.000 n=10) geomean 26.14Mi 19.06Mi -27.07% │ base.txt │ buffer-pool.txt │ │ allocs/op │ allocs/op vs base │ DiffStack/1_x_2_B-10 361.0 ± 0% 326.0 ± 0% -9.70% (p=0.000 n=10) DiffStack/2_x_2_B-10 649.0 ± 0% 621.0 ± 0% -4.31% (p=0.000 n=10) DiffStack/4_x_2_B-10 1.512k ± 0% 1.399k ± 0% -7.47% (p=0.000 n=10) DiffStack/8_x_2_B-10 4.141k ± 0% 4.006k ± 0% -3.25% (p=0.000 n=10) DiffStack/16_x_2_B-10 13.09k ± 0% 12.73k ± 0% -2.72% (p=0.000 n=10) DiffStack/32_x_2_B-10 46.38k ± 0% 44.88k ± 0% -3.25% (p=0.000 n=10) DiffStack/48_x_2_B-10 94.38k ± 0% 87.94k ± 0% -6.82% (p=0.000 n=10) DiffStack/64_x_2_B-10 152.7k ± 0% 149.3k ± 0% -2.18% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 323.0 ± 0% 335.0 ± 0% +3.72% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 630.0 ± 0% 613.0 ± 0% -2.70% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1.354k ± 0% 1.279k ± 0% -5.54% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 3.606k ± 0% 3.490k ± 0% -3.22% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 11.16k ± 0% 10.71k ± 0% -4.11% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 36.45k ± 0% 34.06k ± 0% -6.54% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 71.55k ± 0% 76.00k ± 0% +6.22% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 123.9k ± 0% 125.2k ± 0% +1.07% (p=0.000 n=10) DiffStack/1_x_33_kB-10 346.0 ± 0% 329.0 ± 0% -4.91% (p=0.000 n=10) DiffStack/2_x_33_kB-10 607.0 ± 0% 594.0 ± 0% -2.14% (p=0.000 n=10) DiffStack/4_x_33_kB-10 1.396k ± 0% 1.310k ± 0% -6.16% (p=0.000 n=10) DiffStack/8_x_33_kB-10 3.764k ± 0% 3.564k ± 0% -5.30% (p=0.000 n=10) DiffStack/16_x_33_kB-10 11.26k ± 0% 10.15k ± 0% -9.88% (p=0.000 n=10) DiffStack/32_x_33_kB-10 34.55k ± 0% 34.91k ± 0% +1.03% (p=0.000 n=10) DiffStack/48_x_33_kB-10 69.72k ± 0% 67.76k ± 0% -2.82% (p=0.000 n=10) DiffStack/64_x_33_kB-10 114.1k ± 0% 110.1k ± 0% -3.46% (p=0.000 n=10) DiffStack/2_x_131_kB-10 678.0 ± 0% 643.0 ± 0% -5.16% (p=0.000 n=10) DiffStack/4_x_131_kB-10 1.494k ± 0% 1.419k ± 0% -4.99% (p=0.000 n=10) DiffStack/8_x_131_kB-10 3.737k ± 0% 3.489k ± 0% -6.64% (p=0.000 n=10) DiffStack/16_x_131_kB-10 10.400k ± 0% 9.917k ± 0% -4.64% (p=0.000 n=10) DiffStack/32_x_131_kB-10 32.24k ± 0% 31.08k ± 0% -3.59% (p=0.000 n=10) DiffStack/48_x_131_kB-10 66.12k ± 0% 63.62k ± 0% -3.79% (p=0.000 n=10) DiffStack/64_x_131_kB-10 110.9k ± 0% 109.1k ± 0% -1.67% (p=0.000 n=10) DiffStack/1_x_524_kB-10 374.0 ± 1% 356.0 ± 0% -4.81% (p=0.000 n=10) DiffStack/2_x_524_kB-10 707.0 ± 0% 668.0 ± 0% -5.52% (p=0.000 n=10) DiffStack/4_x_524_kB-10 1.528k ± 0% 1.460k ± 1% -4.48% (p=0.000 n=10) DiffStack/8_x_524_kB-10 3.797k ± 1% 3.571k ± 0% -5.95% (p=0.000 n=10) DiffStack/16_x_524_kB-10 10.562k ± 1% 9.966k ± 0% -5.65% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 512.1k ± 0% 512.1k ± 0% ~ (p=0.222 n=10) DiffStackRecorded/checkpoints.json-10 7.240M ± 0% 7.237M ± 0% -0.03% (p=0.000 n=10) geomean 8.309k 7.996k -3.77% ```
2023-05-05 00:12:34 +00:00
func MarshalUntypedDeployment(b *bytes.Buffer, deployment *apitype.DeploymentV3) error {
all: Reformat with gofumpt Per team discussion, switching to gofumpt. [gofumpt][1] is an alternative, stricter alternative to gofmt. It addresses other stylistic concerns that gofmt doesn't yet cover. [1]: https://github.com/mvdan/gofumpt See the full list of [Added rules][2], but it includes: - Dropping empty lines around function bodies - Dropping unnecessary variable grouping when there's only one variable - Ensuring an empty line between multi-line functions - simplification (`-s` in gofmt) is always enabled - Ensuring multi-line function signatures end with `) {` on a separate line. [2]: https://github.com/mvdan/gofumpt#Added-rules gofumpt is stricter, but there's no lock-in. All gofumpt output is valid gofmt output, so if we decide we don't like it, it's easy to switch back without any code changes. gofumpt support is built into the tooling we use for development so this won't change development workflows. - golangci-lint includes a gofumpt check (enabled in this PR) - gopls, the LSP for Go, includes a gofumpt option (see [installation instrutions][3]) [3]: https://github.com/mvdan/gofumpt#installation This change was generated by running: ```bash gofumpt -w $(rg --files -g '*.go' | rg -v testdata | rg -v compilation_error) ``` The following files were manually tweaked afterwards: - pkg/cmd/pulumi/stack_change_secrets_provider.go: one of the lines overflowed and had comments in an inconvenient place - pkg/cmd/pulumi/destroy.go: `var x T = y` where `T` wasn't necessary - pkg/cmd/pulumi/policy_new.go: long line because of error message - pkg/backend/snapshot_test.go: long line trying to assign three variables in the same assignment I have included mention of gofumpt in the CONTRIBUTING.md.
2023-03-03 16:36:39 +00:00
md := &marshalUntypedDeployment{deployment}
[snapshot] Pool checkpoint buffers Pool the buffers used for marshaling checkpoints. The pooling works by returning the last saved checkpoint's buffer to the diff state when a new checkpoint is saved. If a pooled buffer is available, it is reused when a checkpoint is marshaled. With our current usage patterns, this will cycle through two buffers. Starting from a fresh diff state: - The first call to MarshalDeployment will create a new buffer B1 - The first call to Saved will put B1 into lastSavedDeployment - The second call to MarshalDeployment will create a new buffer B2 - The second call to Saved will put B1 into the pool and B2 into lastSavedDeployment - The third call to MarshalDeployment will get B1 from the pool - The third call to Saved will put B2 into the pool and B1 into lastSavedDeployment These changes reduce CPU time and allocation volume by a geomean of 6% and 27%, respectively. Benchmark results and analysis: httpstate ❯ go test -count=10 -run none -benchmem -bench . | tee buffer-pool.txt httpstate ❯ benchstat base.txt buffer-pool.txt >stat.txt ``` │ base.txt │ buffer-pool.txt │ │ sec/op │ sec/op vs base │ DiffStack/1_x_2_B-10 46.30µ ± 1% 43.17µ ± 0% -6.74% (p=0.000 n=10) DiffStack/2_x_2_B-10 82.68µ ± 1% 80.07µ ± 0% -3.16% (p=0.000 n=10) DiffStack/4_x_2_B-10 191.0µ ± 1% 175.3µ ± 1% -8.22% (p=0.000 n=10) DiffStack/8_x_2_B-10 499.7µ ± 0% 470.6µ ± 1% -5.83% (p=0.000 n=10) DiffStack/16_x_2_B-10 1.489m ± 0% 1.400m ± 1% -5.92% (p=0.000 n=10) DiffStack/32_x_2_B-10 4.895m ± 1% 4.540m ± 0% -7.25% (p=0.000 n=10) DiffStack/48_x_2_B-10 9.651m ± 1% 8.538m ± 1% -11.53% (p=0.000 n=10) DiffStack/64_x_2_B-10 15.11m ± 0% 13.82m ± 0% -8.56% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 103.3µ ± 1% 123.9µ ± 1% +19.93% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 308.8µ ± 0% 317.1µ ± 1% +2.70% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 899.2µ ± 0% 835.6µ ± 0% -7.07% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 2.989m ± 1% 2.757m ± 1% -7.76% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.528m ± 0% 9.305m ± 1% -11.61% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 36.27m ± 1% 30.55m ± 1% -15.77% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 72.53m ± 0% 69.05m ± 0% -4.80% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 125.5m ± 0% 113.5m ± 0% -9.58% (p=0.000 n=10) DiffStack/1_x_33_kB-10 317.4µ ± 1% 314.2µ ± 0% -1.01% (p=0.001 n=10) DiffStack/2_x_33_kB-10 756.8µ ± 1% 797.3µ ± 1% +5.35% (p=0.000 n=10) DiffStack/4_x_33_kB-10 2.605m ± 0% 2.371m ± 0% -8.99% (p=0.000 n=10) DiffStack/8_x_33_kB-10 9.241m ± 2% 8.342m ± 0% -9.73% (p=0.000 n=10) DiffStack/16_x_33_kB-10 32.86m ± 1% 27.11m ± 1% -17.50% (p=0.000 n=10) DiffStack/32_x_33_kB-10 110.2m ± 2% 101.3m ± 1% -8.01% (p=0.000 n=10) DiffStack/48_x_33_kB-10 231.3m ± 0% 202.1m ± 1% -12.64% (p=0.000 n=10) DiffStack/64_x_33_kB-10 383.8m ± 1% 333.4m ± 1% -13.14% (p=0.000 n=10) DiffStack/2_x_131_kB-10 2.748m ± 1% 2.678m ± 0% -2.52% (p=0.000 n=10) DiffStack/4_x_131_kB-10 8.609m ± 0% 8.278m ± 0% -3.85% (p=0.000 n=10) DiffStack/8_x_131_kB-10 28.44m ± 0% 25.46m ± 0% -10.49% (p=0.000 n=10) DiffStack/16_x_131_kB-10 96.98m ± 1% 88.22m ± 0% -9.03% (p=0.000 n=10) DiffStack/32_x_131_kB-10 349.4m ± 1% 315.3m ± 0% -9.74% (p=0.000 n=10) DiffStack/48_x_131_kB-10 761.7m ± 0% 677.9m ± 1% -11.00% (p=0.000 n=10) DiffStack/64_x_131_kB-10 1.312 ± 1% 1.195 ± 0% -8.90% (p=0.000 n=10) DiffStack/1_x_524_kB-10 3.238m ± 1% 3.213m ± 0% -0.78% (p=0.000 n=10) DiffStack/2_x_524_kB-10 9.867m ± 0% 9.613m ± 0% -2.58% (p=0.000 n=10) DiffStack/4_x_524_kB-10 29.65m ± 0% 29.11m ± 0% -1.83% (p=0.000 n=10) DiffStack/8_x_524_kB-10 98.91m ± 1% 92.23m ± 0% -6.76% (p=0.000 n=10) DiffStack/16_x_524_kB-10 348.1m ± 1% 322.5m ± 1% -7.34% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 46.62m ± 0% 46.42m ± 1% -0.42% (p=0.004 n=10) DiffStackRecorded/checkpoints.json-10 835.8m ± 0% 809.5m ± 0% -3.15% (p=0.000 n=10) geomean 11.15m 10.44m -6.39% │ base.txt │ buffer-pool.txt │ │ ratio │ ratio vs base │ DiffStack/1_x_2_B-10 789.5m ± 0% 700.9m ± 0% -11.22% (p=0.000 n=10) DiffStack/2_x_2_B-10 1.034 ± 0% 1.030 ± 0% -0.39% (p=0.000 n=10) DiffStack/4_x_2_B-10 1.676 ± 0% 1.608 ± 0% -4.06% (p=0.000 n=10) DiffStack/8_x_2_B-10 2.883 ± 0% 2.879 ± 0% -0.14% (p=0.000 n=10) DiffStack/16_x_2_B-10 5.270 ± 0% 5.268 ± 0% -0.04% (p=0.000 n=10) DiffStack/32_x_2_B-10 10.20 ± 0% 10.03 ± 0% -1.67% (p=0.000 n=10) DiffStack/48_x_2_B-10 14.24 ± 0% 13.39 ± 0% -5.97% (p=0.000 n=10) DiffStack/64_x_2_B-10 17.51 ± 0% 17.34 ± 0% -0.97% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 950.9m ± 0% 1383.0m ± 0% +45.44% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 1.716 ± 0% 1.900 ± 0% +10.72% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 2.792 ± 0% 2.792 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/8_x_8.2_kB-10 5.350 ± 0% 5.516 ± 0% +3.10% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.62 ± 0% 10.60 ± 0% -0.19% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 20.01 ± 0% 19.10 ± 0% -4.55% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 27.48 ± 0% 30.24 ± 0% +10.04% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 36.97 ± 0% 38.22 ± 0% +3.38% (p=0.000 n=10) DiffStack/1_x_33_kB-10 1.467 ± 0% 1.467 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/2_x_33_kB-10 1.581 ± 0% 1.777 ± 0% +12.40% (p=0.000 n=10) DiffStack/4_x_33_kB-10 2.967 ± 0% 2.967 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/8_x_33_kB-10 5.766 ± 0% 5.809 ± 0% +0.75% (p=0.000 n=10) DiffStack/16_x_33_kB-10 10.87 ± 0% 10.05 ± 0% -7.54% (p=0.000 n=10) DiffStack/32_x_33_kB-10 19.00 ± 0% 20.13 ± 0% +5.95% (p=0.000 n=10) DiffStack/48_x_33_kB-10 27.14 ± 0% 27.18 ± 0% +0.15% (p=0.000 n=10) DiffStack/64_x_33_kB-10 34.39 ± 0% 33.97 ± 0% -1.22% (p=0.000 n=10) DiffStack/2_x_131_kB-10 1.993 ± 0% 1.993 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/4_x_131_kB-10 3.173 ± 0% 3.263 ± 0% +2.84% (p=0.000 n=10) DiffStack/8_x_131_kB-10 5.334 ± 0% 5.291 ± 0% -0.81% (p=0.000 n=10) DiffStack/16_x_131_kB-10 9.296 ± 0% 9.423 ± 0% +1.37% (p=0.000 n=10) DiffStack/32_x_131_kB-10 17.10 ± 0% 17.21 ± 0% +0.64% (p=0.000 n=10) DiffStack/48_x_131_kB-10 25.25 ± 0% 25.06 ± 0% -0.75% (p=0.000 n=10) DiffStack/64_x_131_kB-10 33.13 ± 0% 33.46 ± 0% +1.00% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1.498 ± 0% 1.498 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/2_x_524_kB-10 1.998 ± 0% 1.998 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/4_x_524_kB-10 2.998 ± 0% 3.089 ± 0% +3.04% (p=0.000 n=10) DiffStack/8_x_524_kB-10 5.084 ± 0% 5.084 ± 0% ~ (p=1.000 n=10) ¹ DiffStack/16_x_524_kB-10 9.079 ± 0% 9.058 ± 0% -0.23% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 1.997 ± 0% 1.997 ± 0% ~ (p=1.000 n=10) ¹ DiffStackRecorded/checkpoints.json-10 36.60 ± 0% 36.60 ± 0% ~ (p=1.000 n=10) ¹ geomean 5.961 6.039 +1.31% ¹ all samples are equal │ base.txt │ buffer-pool.txt │ │ B/op │ B/op vs base │ DiffStack/1_x_2_B-10 35.17Ki ± 0% 29.56Ki ± 0% -15.97% (p=0.000 n=10) DiffStack/2_x_2_B-10 70.05Ki ± 0% 61.61Ki ± 0% -12.04% (p=0.000 n=10) DiffStack/4_x_2_B-10 186.8Ki ± 0% 149.9Ki ± 0% -19.75% (p=0.000 n=10) DiffStack/8_x_2_B-10 529.0Ki ± 0% 432.0Ki ± 0% -18.33% (p=0.000 n=10) DiffStack/16_x_2_B-10 1.768Mi ± 0% 1.392Mi ± 0% -21.28% (p=0.000 n=10) DiffStack/32_x_2_B-10 6.513Mi ± 0% 4.986Mi ± 0% -23.45% (p=0.000 n=10) DiffStack/48_x_2_B-10 14.054Mi ± 0% 9.885Mi ± 0% -29.66% (p=0.000 n=10) DiffStack/64_x_2_B-10 22.39Mi ± 0% 16.84Mi ± 0% -24.79% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 154.7Ki ± 0% 202.4Ki ± 0% +30.89% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 571.9Ki ± 0% 540.8Ki ± 0% -5.43% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1.846Mi ± 0% 1.430Mi ± 0% -22.52% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 6.920Mi ± 0% 4.809Mi ± 0% -30.51% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 26.50Mi ± 0% 16.34Mi ± 0% -38.35% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 98.33Mi ± 0% 55.24Mi ± 0% -43.82% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 209.4Mi ± 0% 127.2Mi ± 0% -39.23% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 365.6Mi ± 0% 211.2Mi ± 0% -42.23% (p=0.000 n=10) DiffStack/1_x_33_kB-10 808.7Ki ± 0% 764.2Ki ± 0% -5.50% (p=0.000 n=10) DiffStack/2_x_33_kB-10 2.013Mi ± 0% 1.992Mi ± 0% -1.03% (p=0.000 n=10) DiffStack/4_x_33_kB-10 7.551Mi ± 0% 5.764Mi ± 0% -23.67% (p=0.000 n=10) DiffStack/8_x_33_kB-10 28.58Mi ± 0% 18.90Mi ± 0% -33.87% (p=0.000 n=10) DiffStack/16_x_33_kB-10 104.40Mi ± 0% 59.42Mi ± 0% -43.08% (p=0.000 n=10) DiffStack/32_x_33_kB-10 365.5Mi ± 0% 219.9Mi ± 0% -39.82% (p=0.000 n=10) DiffStack/48_x_33_kB-10 801.1Mi ± 0% 438.7Mi ± 0% -45.24% (p=0.000 n=10) DiffStack/64_x_33_kB-10 1314.3Mi ± 0% 718.3Mi ± 0% -45.35% (p=0.000 n=10) DiffStack/2_x_131_kB-10 9.179Mi ± 0% 8.063Mi ± 0% -12.16% (p=0.000 n=10) DiffStack/4_x_131_kB-10 29.14Mi ± 0% 22.68Mi ± 0% -22.18% (p=0.000 n=10) DiffStack/8_x_131_kB-10 98.77Mi ± 0% 65.16Mi ± 0% -34.03% (p=0.000 n=10) DiffStack/16_x_131_kB-10 337.1Mi ± 0% 207.8Mi ± 0% -38.36% (p=0.000 n=10) DiffStack/32_x_131_kB-10 1234.6Mi ± 0% 705.3Mi ± 0% -42.87% (p=0.000 n=10) DiffStack/48_x_131_kB-10 2.771Gi ± 0% 1.468Gi ± 0% -47.03% (p=0.000 n=10) DiffStack/64_x_131_kB-10 4.606Gi ± 0% 2.553Gi ± 0% -44.56% (p=0.000 n=10) DiffStack/1_x_524_kB-10 12.41Mi ± 1% 11.85Mi ± 1% -4.50% (p=0.000 n=10) DiffStack/2_x_524_kB-10 37.01Mi ± 1% 32.79Mi ± 1% -11.41% (p=0.000 n=10) DiffStack/4_x_524_kB-10 110.56Mi ± 0% 89.08Mi ± 1% -19.43% (p=0.000 n=10) DiffStack/8_x_524_kB-10 367.3Mi ± 1% 254.3Mi ± 1% -30.76% (p=0.000 n=10) DiffStack/16_x_524_kB-10 1291.7Mi ± 0% 801.6Mi ± 0% -37.95% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 69.25Mi ± 0% 69.25Mi ± 0% ~ (p=0.165 n=10) DiffStackRecorded/checkpoints.json-10 1226.7Mi ± 0% 914.2Mi ± 0% -25.47% (p=0.000 n=10) geomean 26.14Mi 19.06Mi -27.07% │ base.txt │ buffer-pool.txt │ │ allocs/op │ allocs/op vs base │ DiffStack/1_x_2_B-10 361.0 ± 0% 326.0 ± 0% -9.70% (p=0.000 n=10) DiffStack/2_x_2_B-10 649.0 ± 0% 621.0 ± 0% -4.31% (p=0.000 n=10) DiffStack/4_x_2_B-10 1.512k ± 0% 1.399k ± 0% -7.47% (p=0.000 n=10) DiffStack/8_x_2_B-10 4.141k ± 0% 4.006k ± 0% -3.25% (p=0.000 n=10) DiffStack/16_x_2_B-10 13.09k ± 0% 12.73k ± 0% -2.72% (p=0.000 n=10) DiffStack/32_x_2_B-10 46.38k ± 0% 44.88k ± 0% -3.25% (p=0.000 n=10) DiffStack/48_x_2_B-10 94.38k ± 0% 87.94k ± 0% -6.82% (p=0.000 n=10) DiffStack/64_x_2_B-10 152.7k ± 0% 149.3k ± 0% -2.18% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 323.0 ± 0% 335.0 ± 0% +3.72% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 630.0 ± 0% 613.0 ± 0% -2.70% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1.354k ± 0% 1.279k ± 0% -5.54% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 3.606k ± 0% 3.490k ± 0% -3.22% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 11.16k ± 0% 10.71k ± 0% -4.11% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 36.45k ± 0% 34.06k ± 0% -6.54% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 71.55k ± 0% 76.00k ± 0% +6.22% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 123.9k ± 0% 125.2k ± 0% +1.07% (p=0.000 n=10) DiffStack/1_x_33_kB-10 346.0 ± 0% 329.0 ± 0% -4.91% (p=0.000 n=10) DiffStack/2_x_33_kB-10 607.0 ± 0% 594.0 ± 0% -2.14% (p=0.000 n=10) DiffStack/4_x_33_kB-10 1.396k ± 0% 1.310k ± 0% -6.16% (p=0.000 n=10) DiffStack/8_x_33_kB-10 3.764k ± 0% 3.564k ± 0% -5.30% (p=0.000 n=10) DiffStack/16_x_33_kB-10 11.26k ± 0% 10.15k ± 0% -9.88% (p=0.000 n=10) DiffStack/32_x_33_kB-10 34.55k ± 0% 34.91k ± 0% +1.03% (p=0.000 n=10) DiffStack/48_x_33_kB-10 69.72k ± 0% 67.76k ± 0% -2.82% (p=0.000 n=10) DiffStack/64_x_33_kB-10 114.1k ± 0% 110.1k ± 0% -3.46% (p=0.000 n=10) DiffStack/2_x_131_kB-10 678.0 ± 0% 643.0 ± 0% -5.16% (p=0.000 n=10) DiffStack/4_x_131_kB-10 1.494k ± 0% 1.419k ± 0% -4.99% (p=0.000 n=10) DiffStack/8_x_131_kB-10 3.737k ± 0% 3.489k ± 0% -6.64% (p=0.000 n=10) DiffStack/16_x_131_kB-10 10.400k ± 0% 9.917k ± 0% -4.64% (p=0.000 n=10) DiffStack/32_x_131_kB-10 32.24k ± 0% 31.08k ± 0% -3.59% (p=0.000 n=10) DiffStack/48_x_131_kB-10 66.12k ± 0% 63.62k ± 0% -3.79% (p=0.000 n=10) DiffStack/64_x_131_kB-10 110.9k ± 0% 109.1k ± 0% -1.67% (p=0.000 n=10) DiffStack/1_x_524_kB-10 374.0 ± 1% 356.0 ± 0% -4.81% (p=0.000 n=10) DiffStack/2_x_524_kB-10 707.0 ± 0% 668.0 ± 0% -5.52% (p=0.000 n=10) DiffStack/4_x_524_kB-10 1.528k ± 0% 1.460k ± 1% -4.48% (p=0.000 n=10) DiffStack/8_x_524_kB-10 3.797k ± 1% 3.571k ± 0% -5.95% (p=0.000 n=10) DiffStack/16_x_524_kB-10 10.562k ± 1% 9.966k ± 0% -5.65% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 512.1k ± 0% 512.1k ± 0% ~ (p=0.222 n=10) DiffStackRecorded/checkpoints.json-10 7.240M ± 0% 7.237M ± 0% -0.03% (p=0.000 n=10) geomean 8.309k 7.996k -3.77% ```
2023-05-05 00:12:34 +00:00
return md.Write(b)
}
[snapshot] Use a newer diff package The version of gopls that gotextdiff is derived from is over three years old. In the intervening time, the differ used by gopls has improved tremendously, and the language has grown new features like generics. These changes use a different diff package that is derived from an up-to-date version of gopls and exposes an API that is parameterized over the storage of the text. Taken together, this allows us to use a much faster and more memory-efficient algorithm for diffing while safely avoiding string copies when invoking the differ. The more efficient algorithm operates on arbitrary slices whose elements are comparable. Naive use of this algorithm--passing byte slices in--can cause increased time spent diffing, as the sequences being compared are longer. In order to avoid this, these changes record spans of marshaled deployments as the deployments are marshaled. These spans are strings that cover the diffable elements of the deployment: its header, its resources, and its pending operations. The output of the newer algorithm is converible to the format we currently send to the service, so these changes should not require service-side changes. In order to avoid bumping the minimum Go version required to build the `github.com/pulumi/pulumi/pkg/v3" and therefore break backwards compatibility with consumers using older toolchains (e.g. providers), these changes use the existing differ when built with a pre-1.20 toolchain. Official builds of the CLI are already using Go 1.20+ and should use the new differ without additional changes. These changes improve CPU time, allocation volume, and the compression ratio by 57%, 87%, and 52%, respectively. Benchmark results and analysis: httpstate ❯ go test -count=10 -run none -benchmem -bench . | tee spanned.txt httpstate ❯ benchstat buffer-pool.txt spanned.txt >stat.txt ``` │ buffer-pool.txt │ spanned.txt │ │ sec/op │ sec/op vs base │ DiffStack/1_x_2_B-10 45.09µ ± 1% 22.81µ ± 1% -49.41% (p=0.000 n=10) DiffStack/2_x_2_B-10 79.49µ ± 1% 40.60µ ± 1% -48.93% (p=0.000 n=10) DiffStack/4_x_2_B-10 172.54µ ± 2% 81.57µ ± 0% -52.72% (p=0.000 n=10) DiffStack/8_x_2_B-10 455.4µ ± 1% 200.4µ ± 0% -56.00% (p=0.000 n=10) DiffStack/16_x_2_B-10 1388.2µ ± 1% 575.1µ ± 1% -58.57% (p=0.000 n=10) DiffStack/32_x_2_B-10 4.409m ± 1% 1.866m ± 0% -57.67% (p=0.000 n=10) DiffStack/48_x_2_B-10 9.275m ± 0% 3.484m ± 0% -62.44% (p=0.000 n=10) DiffStack/64_x_2_B-10 14.564m ± 0% 5.945m ± 1% -59.18% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 124.81µ ± 1% 40.14µ ± 1% -67.84% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 299.8µ ± 0% 121.8µ ± 1% -59.38% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 846.2µ ± 1% 355.8µ ± 0% -57.96% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 2.743m ± 1% 1.150m ± 0% -58.06% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 9.271m ± 1% 4.041m ± 0% -56.41% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 31.78m ± 0% 14.97m ± 0% -52.88% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 66.14m ± 1% 30.66m ± 0% -53.63% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 107.50m ± 0% 53.44m ± 0% -50.29% (p=0.000 n=10) DiffStack/1_x_33_kB-10 240.51µ ± 1% 76.95µ ± 0% -68.01% (p=0.000 n=10) DiffStack/2_x_33_kB-10 798.5µ ± 0% 310.6µ ± 0% -61.10% (p=0.000 n=10) DiffStack/4_x_33_kB-10 2451.9µ ± 1% 978.9µ ± 3% -60.08% (p=0.000 n=10) DiffStack/8_x_33_kB-10 8.258m ± 0% 3.731m ± 0% -54.82% (p=0.000 n=10) DiffStack/16_x_33_kB-10 28.29m ± 1% 12.77m ± 0% -54.85% (p=0.000 n=10) DiffStack/32_x_33_kB-10 104.11m ± 1% 44.84m ± 1% -56.93% (p=0.000 n=10) DiffStack/48_x_33_kB-10 199.73m ± 1% 95.87m ± 1% -52.00% (p=0.000 n=10) DiffStack/64_x_33_kB-10 339.6m ± 2% 159.2m ± 2% -53.12% (p=0.000 n=10) DiffStack/2_x_131_kB-10 2721.7µ ± 0% 984.3µ ± 0% -63.84% (p=0.000 n=10) DiffStack/4_x_131_kB-10 8.334m ± 1% 3.654m ± 2% -56.15% (p=0.000 n=10) DiffStack/8_x_131_kB-10 25.89m ± 0% 12.23m ± 0% -52.77% (p=0.000 n=10) DiffStack/16_x_131_kB-10 87.49m ± 1% 40.26m ± 0% -53.98% (p=0.000 n=10) DiffStack/32_x_131_kB-10 317.0m ± 5% 150.4m ± 0% -52.57% (p=0.000 n=10) DiffStack/48_x_131_kB-10 681.8m ± 1% 332.1m ± 0% -51.28% (p=0.000 n=10) DiffStack/64_x_131_kB-10 1190.4m ± 1% 583.1m ± 0% -51.01% (p=0.000 n=10) DiffStack/1_x_524_kB-10 3.215m ± 0% 1.086m ± 0% -66.21% (p=0.000 n=10) DiffStack/2_x_524_kB-10 9.676m ± 1% 3.994m ± 0% -58.72% (p=0.000 n=10) DiffStack/4_x_524_kB-10 29.84m ± 0% 12.88m ± 1% -56.84% (p=0.000 n=10) DiffStack/8_x_524_kB-10 92.11m ± 1% 42.35m ± 0% -54.02% (p=0.000 n=10) DiffStack/16_x_524_kB-10 322.1m ± 0% 154.0m ± 0% -52.19% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 54.73m ± 1% 22.63m ± 0% -58.65% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 815.1m ± 0% 395.5m ± 2% -51.48% (p=0.000 n=10) geomean 10.45m 4.530m -56.65% │ buffer-pool.txt │ spanned.txt │ │ checkpoint_bytes │ checkpoint_bytes vs base │ DiffStack/1_x_2_B-10 2.539k ± 0% 1.950k ± 0% -23.20% (p=0.000 n=10) DiffStack/2_x_2_B-10 5.201k ± 0% 4.548k ± 0% -12.56% (p=0.000 n=10) DiffStack/4_x_2_B-10 13.63k ± 0% 12.30k ± 0% -9.77% (p=0.000 n=10) DiffStack/8_x_2_B-10 45.09k ± 0% 42.86k ± 0% -4.95% (p=0.000 n=10) DiffStack/16_x_2_B-10 161.9k ± 0% 156.4k ± 0% -3.42% (p=0.000 n=10) DiffStack/32_x_2_B-10 579.8k ± 0% 604.2k ± 0% +4.21% (p=0.000 n=10) DiffStack/48_x_2_B-10 1.306M ± 0% 1.180M ± 0% -9.68% (p=0.000 n=10) DiffStack/64_x_2_B-10 2.164M ± 0% 2.134M ± 0% -1.36% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 27.109k ± 0% 9.856k ± 0% -63.64% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 78.91k ± 0% 61.59k ± 0% -21.94% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 284.2k ± 0% 258.3k ± 0% -9.13% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 1.119M ± 0% 1.041M ± 0% -6.89% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 4.389M ± 0% 4.201M ± 0% -4.29% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 16.74M ± 0% 17.06M ± 0% +1.95% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 36.30M ± 0% 35.93M ± 0% -1.00% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 60.98M ± 0% 64.06M ± 0% +5.05% (p=0.000 n=10) DiffStack/1_x_33_kB-10 67.79k ± 0% 34.43k ± 0% -49.20% (p=0.000 n=10) DiffStack/2_x_33_kB-10 300.1k ± 0% 233.6k ± 0% -22.15% (p=0.000 n=10) DiffStack/4_x_33_kB-10 1128.3k ± 0% 929.5k ± 0% -17.62% (p=0.000 n=10) DiffStack/8_x_33_kB-10 4.437M ± 0% 4.238M ± 0% -4.49% (p=0.000 n=10) DiffStack/16_x_33_kB-10 16.57M ± 0% 15.78M ± 0% -4.79% (p=0.000 n=10) DiffStack/32_x_33_kB-10 64.35M ± 0% 58.66M ± 0% -8.84% (p=0.000 n=10) DiffStack/48_x_33_kB-10 127.0M ± 0% 128.0M ± 0% +0.80% (p=0.000 n=10) DiffStack/64_x_33_kB-10 217.7M ± 0% 216.5M ± 0% -0.54% (p=0.000 n=10) DiffStack/2_x_131_kB-10 1316.2k ± 0% 921.8k ± 0% -29.97% (p=0.000 n=10) DiffStack/4_x_131_kB-10 4.733M ± 0% 4.207M ± 0% -11.11% (p=0.000 n=10) DiffStack/8_x_131_kB-10 16.03M ± 0% 15.77M ± 0% -1.64% (p=0.000 n=10) DiffStack/16_x_131_kB-10 57.42M ± 0% 54.66M ± 0% -4.81% (p=0.000 n=10) DiffStack/32_x_131_kB-10 214.8M ± 0% 210.7M ± 0% -1.90% (p=0.000 n=10) DiffStack/48_x_131_kB-10 472.9M ± 0% 466.9M ± 0% -1.28% (p=0.000 n=10) DiffStack/64_x_131_kB-10 832.3M ± 0% 824.7M ± 0% -0.92% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1.575M ± 0% 1.051M ± 0% -33.32% (p=0.000 n=10) DiffStack/2_x_524_kB-10 5.248M ± 0% 4.199M ± 0% -20.00% (p=0.000 n=10) DiffStack/4_x_524_kB-10 18.36M ± 0% 15.74M ± 0% -14.28% (p=0.000 n=10) DiffStack/8_x_524_kB-10 60.86M ± 0% 56.66M ± 0% -6.90% (p=0.000 n=10) DiffStack/16_x_524_kB-10 223.5M ± 0% 214.0M ± 0% -4.23% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 13.21M ± 0% 13.20M ± 0% -0.01% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 155.9M ± 0% 154.9M ± 0% -0.60% (p=0.000 n=10) geomean 3.606M 3.173M -12.01% │ buffer-pool.txt │ spanned.txt │ │ ratio │ ratio vs base │ DiffStack/1_x_2_B-10 789.5m ± 0% 1641.0m ± 0% +107.85% (p=0.000 n=10) DiffStack/2_x_2_B-10 1.034 ± 0% 2.353 ± 0% +127.56% (p=0.000 n=10) DiffStack/4_x_2_B-10 1.575 ± 0% 3.592 ± 0% +128.06% (p=0.000 n=10) DiffStack/8_x_2_B-10 2.829 ± 0% 6.684 ± 0% +136.27% (p=0.000 n=10) DiffStack/16_x_2_B-10 5.296 ± 0% 12.610 ± 0% +138.10% (p=0.000 n=10) DiffStack/32_x_2_B-10 9.681 ± 0% 24.790 ± 0% +156.07% (p=0.000 n=10) DiffStack/48_x_2_B-10 14.65 ± 0% 32.44 ± 0% +121.43% (p=0.000 n=10) DiffStack/64_x_2_B-10 18.26 ± 0% 44.11 ± 0% +141.57% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 1.383 ± 0% 1.051 ± 0% -24.01% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 1.716 ± 0% 2.324 ± 0% +35.43% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 2.878 ± 0% 4.251 ± 0% +47.71% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 5.475 ± 0% 8.056 ± 0% +47.14% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.56 ± 0% 15.77 ± 0% +49.34% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 19.97 ± 0% 31.57 ± 0% +58.09% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 28.80 ± 0% 44.11 ± 0% +53.16% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 36.23 ± 0% 58.84 ± 0% +62.41% (p=0.000 n=10) DiffStack/1_x_33_kB-10 986.0m ± 0% 1014.0m ± 0% +2.84% (p=0.000 n=10) DiffStack/2_x_33_kB-10 1.777 ± 0% 2.331 ± 0% +31.18% (p=0.000 n=10) DiffStack/4_x_33_kB-10 3.057 ± 0% 3.993 ± 0% +30.62% (p=0.000 n=10) DiffStack/8_x_33_kB-10 5.766 ± 0% 8.511 ± 0% +47.61% (p=0.000 n=10) DiffStack/16_x_33_kB-10 10.55 ± 0% 15.35 ± 0% +45.50% (p=0.000 n=10) DiffStack/32_x_33_kB-10 20.28 ± 0% 28.08 ± 0% +38.46% (p=0.000 n=10) DiffStack/48_x_33_kB-10 26.59 ± 0% 40.64 ± 0% +52.84% (p=0.000 n=10) DiffStack/64_x_33_kB-10 34.14 ± 0% 51.44 ± 0% +50.67% (p=0.000 n=10) DiffStack/2_x_131_kB-10 1.993 ± 0% 2.333 ± 0% +17.06% (p=0.000 n=10) DiffStack/4_x_131_kB-10 3.263 ± 0% 4.569 ± 0% +40.02% (p=0.000 n=10) DiffStack/8_x_131_kB-10 5.291 ± 0% 7.995 ± 0% +51.11% (p=0.000 n=10) DiffStack/16_x_131_kB-10 9.275 ± 0% 13.410 ± 0% +44.58% (p=0.000 n=10) DiffStack/32_x_131_kB-10 17.17 ± 0% 25.44 ± 0% +48.17% (p=0.000 n=10) DiffStack/48_x_131_kB-10 25.11 ± 0% 37.39 ± 0% +48.90% (p=0.000 n=10) DiffStack/64_x_131_kB-10 33.09 ± 0% 49.40 ± 0% +49.29% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1.498 ± 0% 1.999 ± 0% +33.44% (p=0.000 n=10) DiffStack/2_x_524_kB-10 1.998 ± 0% 2.666 ± 0% +33.43% (p=0.000 n=10) DiffStack/4_x_524_kB-10 3.180 ± 0% 4.285 ± 0% +34.75% (p=0.000 n=10) DiffStack/8_x_524_kB-10 5.040 ± 0% 7.199 ± 0% +42.84% (p=0.000 n=10) DiffStack/16_x_524_kB-10 9.058 ± 0% 13.160 ± 0% +45.29% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 1.997 ± 0% 1.999 ± 0% +0.10% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 36.60 ± 0% 40.42 ± 0% +10.44% (p=0.000 n=10) geomean 5.993 9.122 +52.20% │ buffer-pool.txt │ spanned.txt │ │ wire_bytes │ wire_bytes vs base │ DiffStack/1_x_2_B-10 3.216k ± 0% 1.188k ± 0% -63.06% (p=0.000 n=10) DiffStack/2_x_2_B-10 5.031k ± 0% 1.933k ± 0% -61.58% (p=0.000 n=10) DiffStack/4_x_2_B-10 8.655k ± 0% 3.424k ± 0% -60.44% (p=0.000 n=10) DiffStack/8_x_2_B-10 15.938k ± 0% 6.412k ± 0% -59.77% (p=0.000 n=10) DiffStack/16_x_2_B-10 30.58k ± 0% 12.40k ± 0% -59.45% (p=0.000 n=10) DiffStack/32_x_2_B-10 59.89k ± 0% 24.37k ± 0% -59.31% (p=0.000 n=10) DiffStack/48_x_2_B-10 89.17k ± 0% 36.38k ± 0% -59.21% (p=0.000 n=10) DiffStack/64_x_2_B-10 118.48k ± 0% 48.39k ± 0% -59.16% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 19.596k ± 0% 9.378k ± 0% -52.14% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 45.98k ± 0% 26.50k ± 0% -42.36% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 98.75k ± 0% 60.76k ± 0% -38.47% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 204.3k ± 0% 129.3k ± 0% -36.72% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 415.5k ± 0% 266.3k ± 0% -35.91% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 837.9k ± 0% 540.4k ± 0% -35.50% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 1260.4k ± 0% 814.6k ± 0% -35.37% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 1.683M ± 0% 1.089M ± 0% -35.31% (p=0.000 n=10) DiffStack/1_x_33_kB-10 68.75k ± 0% 33.95k ± 0% -50.61% (p=0.000 n=10) DiffStack/2_x_33_kB-10 168.9k ± 0% 100.2k ± 0% -40.64% (p=0.000 n=10) DiffStack/4_x_33_kB-10 369.1k ± 0% 232.8k ± 0% -36.93% (p=0.000 n=10) DiffStack/8_x_33_kB-10 769.6k ± 0% 497.9k ± 0% -35.30% (p=0.000 n=10) DiffStack/16_x_33_kB-10 1.571M ± 0% 1.028M ± 0% -34.53% (p=0.000 n=10) DiffStack/32_x_33_kB-10 3.173M ± 0% 2.089M ± 0% -34.16% (p=0.000 n=10) DiffStack/48_x_33_kB-10 4.775M ± 0% 3.149M ± 0% -34.04% (p=0.000 n=10) DiffStack/64_x_33_kB-10 6.377M ± 0% 4.210M ± 0% -33.98% (p=0.000 n=10) DiffStack/2_x_131_kB-10 660.4k ± 0% 395.1k ± 0% -40.16% (p=0.000 n=10) DiffStack/4_x_131_kB-10 1450.4k ± 0% 920.9k ± 0% -36.51% (p=0.000 n=10) DiffStack/8_x_131_kB-10 3.031M ± 0% 1.972M ± 0% -34.91% (p=0.000 n=10) DiffStack/16_x_131_kB-10 6.191M ± 0% 4.076M ± 0% -34.17% (p=0.000 n=10) DiffStack/32_x_131_kB-10 12.512M ± 0% 8.282M ± 0% -33.81% (p=0.000 n=10) DiffStack/48_x_131_kB-10 18.83M ± 0% 12.49M ± 0% -33.69% (p=0.000 n=10) DiffStack/64_x_131_kB-10 25.15M ± 0% 16.69M ± 0% -33.63% (p=0.000 n=10) DiffStack/1_x_524_kB-10 1051.8k ± 0% 525.5k ± 0% -50.04% (p=0.000 n=10) DiffStack/2_x_524_kB-10 2.626M ± 0% 1.575M ± 0% -40.04% (p=0.000 n=10) DiffStack/4_x_524_kB-10 5.776M ± 0% 3.673M ± 0% -36.40% (p=0.000 n=10) DiffStack/8_x_524_kB-10 12.075M ± 0% 7.871M ± 0% -34.82% (p=0.000 n=10) DiffStack/16_x_524_kB-10 24.67M ± 0% 16.27M ± 0% -34.07% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 6.612M ± 0% 6.606M ± 0% -0.10% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 4.259M ± 0% 3.833M ± 0% -10.00% (p=0.000 n=10) geomean 601.7k 347.8k -42.19% │ buffer-pool.txt │ spanned.txt │ │ B/op │ B/op vs base │ DiffStack/1_x_2_B-10 31.85Ki ± 0% 15.79Ki ± 0% -50.42% (p=0.000 n=10) DiffStack/2_x_2_B-10 61.81Ki ± 0% 28.01Ki ± 0% -54.68% (p=0.000 n=10) DiffStack/4_x_2_B-10 147.58Ki ± 0% 55.60Ki ± 0% -62.33% (p=0.000 n=10) DiffStack/8_x_2_B-10 425.9Ki ± 0% 116.9Ki ± 0% -72.55% (p=0.000 n=10) DiffStack/16_x_2_B-10 1429.9Ki ± 0% 307.8Ki ± 0% -78.47% (p=0.000 n=10) DiffStack/32_x_2_B-10 4955.7Ki ± 0% 934.1Ki ± 0% -81.15% (p=0.000 n=10) DiffStack/48_x_2_B-10 10.746Mi ± 0% 1.682Mi ± 0% -84.34% (p=0.000 n=10) DiffStack/64_x_2_B-10 17.664Mi ± 0% 2.822Mi ± 0% -84.03% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 202.50Ki ± 0% 41.18Ki ± 0% -79.66% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 511.5Ki ± 0% 134.1Ki ± 0% -73.78% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1513.5Ki ± 0% 309.4Ki ± 0% -79.56% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 4886.0Ki ± 0% 674.2Ki ± 0% -86.20% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 16.287Mi ± 0% 1.427Mi ± 0% -91.24% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 57.541Mi ± 0% 3.198Mi ± 0% -94.44% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 121.703Mi ± 0% 5.810Mi ± 0% -95.23% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 200.588Mi ± 0% 7.495Mi ± 0% -96.26% (p=0.000 n=10) DiffStack/1_x_33_kB-10 562.5Ki ± 0% 136.3Ki ± 0% -75.78% (p=0.000 n=10) DiffStack/2_x_33_kB-10 2041.0Ki ± 0% 505.5Ki ± 0% -75.23% (p=0.000 n=10) DiffStack/4_x_33_kB-10 5.874Mi ± 0% 1.129Mi ± 0% -80.78% (p=0.000 n=10) DiffStack/8_x_33_kB-10 18.837Mi ± 0% 2.410Mi ± 0% -87.21% (p=0.000 n=10) DiffStack/16_x_33_kB-10 61.898Mi ± 0% 5.015Mi ± 0% -91.90% (p=0.000 n=10) DiffStack/32_x_33_kB-10 221.69Mi ± 0% 10.36Mi ± 0% -95.32% (p=0.000 n=10) DiffStack/48_x_33_kB-10 430.13Mi ± 0% 18.50Mi ± 0% -95.70% (p=0.000 n=10) DiffStack/64_x_33_kB-10 722.07Mi ± 0% 21.63Mi ± 0% -97.00% (p=0.000 n=10) DiffStack/2_x_131_kB-10 8.098Mi ± 0% 1.656Mi ± 0% -79.55% (p=0.000 n=10) DiffStack/4_x_131_kB-10 22.662Mi ± 0% 3.934Mi ± 0% -82.64% (p=0.000 n=10) DiffStack/8_x_131_kB-10 65.217Mi ± 0% 8.425Mi ± 0% -87.08% (p=0.000 n=10) DiffStack/16_x_131_kB-10 204.90Mi ± 0% 17.22Mi ± 1% -91.60% (p=0.000 n=10) DiffStack/32_x_131_kB-10 703.12Mi ± 0% 34.89Mi ± 0% -95.04% (p=0.000 n=10) DiffStack/48_x_131_kB-10 1505.36Mi ± 0% 61.24Mi ± 0% -95.93% (p=0.000 n=10) DiffStack/64_x_131_kB-10 2586.04Mi ± 0% 70.48Mi ± 1% -97.27% (p=0.000 n=10) DiffStack/1_x_524_kB-10 11.854Mi ± 1% 2.127Mi ± 0% -82.06% (p=0.000 n=10) DiffStack/2_x_524_kB-10 32.932Mi ± 1% 6.558Mi ± 1% -80.09% (p=0.000 n=10) DiffStack/4_x_524_kB-10 90.04Mi ± 0% 15.15Mi ± 0% -83.17% (p=0.000 n=10) DiffStack/8_x_524_kB-10 252.94Mi ± 0% 32.27Mi ± 1% -87.24% (p=0.000 n=10) DiffStack/16_x_524_kB-10 801.77Mi ± 0% 66.46Mi ± 1% -91.71% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 103.67Mi ± 0% 35.52Mi ± 0% -65.74% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 913.90Mi ± 0% 53.53Mi ± 1% -94.14% (p=0.000 n=10) geomean 19.16Mi 2.409Mi -87.43% │ buffer-pool.txt │ spanned.txt │ │ allocs/op │ allocs/op vs base │ DiffStack/1_x_2_B-10 344.0 ± 0% 184.0 ± 0% -46.51% (p=0.000 n=10) DiffStack/2_x_2_B-10 616.0 ± 0% 295.0 ± 0% -52.11% (p=0.000 n=10) DiffStack/4_x_2_B-10 1381.0 ± 0% 524.0 ± 0% -62.06% (p=0.000 n=10) DiffStack/8_x_2_B-10 3.922k ± 0% 1.032k ± 0% -73.69% (p=0.000 n=10) DiffStack/16_x_2_B-10 12.791k ± 0% 2.233k ± 0% -82.54% (p=0.000 n=10) DiffStack/32_x_2_B-10 43.416k ± 0% 5.410k ± 0% -87.54% (p=0.000 n=10) DiffStack/48_x_2_B-10 95.803k ± 0% 9.036k ± 0% -90.57% (p=0.000 n=10) DiffStack/64_x_2_B-10 156.99k ± 0% 13.99k ± 0% -91.09% (p=0.000 n=10) DiffStack/1_x_8.2_kB-10 335.0 ± 0% 182.0 ± 0% -45.67% (p=0.000 n=10) DiffStack/2_x_8.2_kB-10 597.5 ± 0% 295.0 ± 0% -50.63% (p=0.000 n=10) DiffStack/4_x_8.2_kB-10 1304.0 ± 0% 527.0 ± 0% -59.59% (p=0.000 n=10) DiffStack/8_x_8.2_kB-10 3.475k ± 0% 1.036k ± 0% -70.19% (p=0.000 n=10) DiffStack/16_x_8.2_kB-10 10.677k ± 0% 2.239k ± 0% -79.03% (p=0.000 n=10) DiffStack/32_x_8.2_kB-10 35.357k ± 0% 5.414k ± 0% -84.69% (p=0.000 n=10) DiffStack/48_x_8.2_kB-10 72.782k ± 0% 9.296k ± 0% -87.23% (p=0.000 n=10) DiffStack/64_x_8.2_kB-10 119.24k ± 0% 14.26k ± 0% -88.04% (p=0.000 n=10) DiffStack/1_x_33_kB-10 309.0 ± 0% 182.0 ± 0% -41.10% (p=0.000 n=10) DiffStack/2_x_33_kB-10 594.0 ± 0% 296.0 ± 0% -50.17% (p=0.000 n=10) DiffStack/4_x_33_kB-10 1324.5 ± 0% 528.0 ± 0% -60.14% (p=0.000 n=10) DiffStack/8_x_33_kB-10 3.554k ± 0% 1.047k ± 0% -70.54% (p=0.000 n=10) DiffStack/16_x_33_kB-10 10.511k ± 0% 2.232k ± 0% -78.77% (p=0.000 n=10) DiffStack/32_x_33_kB-10 35.147k ± 0% 5.194k ± 0% -85.22% (p=0.000 n=10) DiffStack/48_x_33_kB-10 66.519k ± 0% 8.958k ± 0% -86.53% (p=0.000 n=10) DiffStack/64_x_33_kB-10 110.71k ± 0% 13.28k ± 0% -88.00% (p=0.000 n=10) DiffStack/2_x_131_kB-10 644.0 ± 0% 296.0 ± 0% -54.04% (p=0.000 n=10) DiffStack/4_x_131_kB-10 1418.5 ± 0% 535.0 ± 0% -62.28% (p=0.000 n=10) DiffStack/8_x_131_kB-10 3.497k ± 0% 1.047k ± 0% -70.06% (p=0.000 n=10) DiffStack/16_x_131_kB-10 9.805k ± 0% 2.182k ± 0% -77.75% (p=0.000 n=10) DiffStack/32_x_131_kB-10 30.996k ± 0% 5.040k ± 0% -83.74% (p=0.000 n=10) DiffStack/48_x_131_kB-10 63.751k ± 0% 8.656k ± 0% -86.42% (p=0.000 n=10) DiffStack/64_x_131_kB-10 107.97k ± 0% 13.03k ± 0% -87.93% (p=0.000 n=10) DiffStack/1_x_524_kB-10 356.0 ± 0% 186.0 ± 0% -47.75% (p=0.000 n=10) DiffStack/2_x_524_kB-10 671.0 ± 0% 300.0 ± 0% -55.29% (p=0.000 n=10) DiffStack/4_x_524_kB-10 1467.5 ± 0% 536.0 ± 0% -63.48% (p=0.000 n=10) DiffStack/8_x_524_kB-10 3.553k ± 0% 1.038k ± 0% -70.80% (p=0.000 n=10) DiffStack/16_x_524_kB-10 9.971k ± 0% 2.182k ± 0% -78.12% (p=0.000 n=10) DiffStackRecorded/two-large-checkpoints.json-10 489.16k ± 0% 73.65k ± 0% -84.94% (p=0.000 n=10) DiffStackRecorded/checkpoints.json-10 7236.9k ± 0% 857.8k ± 0% -88.15% (p=0.000 n=10) geomean 7.989k 1.934k -75.79% ```
2023-05-05 02:15:09 +00:00
func marshalDeployment(d *apitype.DeploymentV3) (json.RawMessage, error) {
var b bytes.Buffer
if err := MarshalUntypedDeployment(&b, d); err != nil {
return nil, err
}
return json.RawMessage(b.Bytes()), nil
}
func marshalVerbatimCheckpointRequest(req apitype.PatchUpdateVerbatimCheckpointRequest) (json.RawMessage, error) {
// Unlike encoding/json, using jsonIter here will not reindent req.UntypedDeployment, which is what is needed
// for the Verbatim protocol.
return jsonIterConfig.Marshal(req)
}
type marshalUntypedDeployment struct {
deployment *apitype.DeploymentV3
}
func (c *marshalUntypedDeployment) Write(w io.Writer) error {
cfg := jsonIterConfig
stream := cfg.BorrowStream(w)
defer cfg.ReturnStream(stream)
err := c.writeToStream(stream)
return err
}
func (c *marshalUntypedDeployment) writeToStream(stream *jsoniter.Stream) error {
2022-12-15 17:12:32 +00:00
stream.WriteObjectStart() // writes `{`
stream.WriteObjectField("version") // writes `"version":`
stream.WriteInt(3)
2022-12-15 17:12:32 +00:00
stream.WriteMore() // writes `,`
stream.WriteObjectField("deployment")
2022-12-15 17:14:51 +00:00
err := c.writeDeploymentV3(stream)
if err != nil {
return err
}
2022-12-15 17:12:32 +00:00
stream.WriteObjectEnd() // writes `}`
return stream.Flush()
}
func (c *marshalUntypedDeployment) writeDeploymentV3(stream *jsoniter.Stream) (err error) {
deployment := c.deployment
stream.WriteObjectStart()
stream.WriteObjectField("manifest")
stream.WriteVal(deployment.Manifest)
if deployment.SecretsProviders != nil {
stream.WriteMore()
stream.WriteObjectField("secrets_providers")
stream.WriteVal(deployment.SecretsProviders)
}
if err = stream.Flush(); err != nil {
return err
}
nResources := len(deployment.Resources)
2022-12-15 17:12:32 +00:00
maxNL := maxNewLines - 2
newlinePeriod := int(math.Ceil(float64(nResources) / float64(maxNL)))
if nResources > 0 {
stream.WriteMore()
stream.WriteObjectField("resources")
stream.WriteRaw("[\n")
for i, r := range deployment.Resources {
if i > 0 {
stream.WriteRaw(",")
2022-12-15 17:12:32 +00:00
if (nResources <= maxNL) || (i%newlinePeriod == 0) {
stream.WriteRaw("\n")
}
}
stream.WriteVal(r)
if err = stream.Flush(); err != nil {
return err
}
}
stream.WriteRaw("\n]")
}
if len(deployment.PendingOperations) > 0 {
stream.WriteMore()
stream.WriteObjectField("pendingOperations")
stream.WriteVal(deployment.PendingOperations)
}
stream.WriteObjectEnd()
return stream.Flush()
}