[snapshot] Use a newer diff package
The version of gopls that gotextdiff is derived from is over three years
old. In the intervening time, the differ used by gopls has improved
tremendously, and the language has grown new features like generics.
These changes use a different diff package that is derived from an
up-to-date version of gopls and exposes an API that is parameterized
over the storage of the text. Taken together, this allows us to use a
much faster and more memory-efficient algorithm for diffing while
safely avoiding string copies when invoking the differ.
The more efficient algorithm operates on arbitrary slices whose elements
are comparable. Naive use of this algorithm--passing byte slices
in--can cause increased time spent diffing, as the sequences being
compared are longer. In order to avoid this, these changes record
spans of marshaled deployments as the deployments are marshaled. These
spans are strings that cover the diffable elements of the deployment:
its header, its resources, and its pending operations.
The output of the newer algorithm is converible to the format we
currently send to the service, so these changes should not require
service-side changes.
In order to avoid bumping the minimum Go version required to build the
`github.com/pulumi/pulumi/pkg/v3" and therefore break backwards
compatibility with consumers using older toolchains (e.g. providers),
these changes use the existing differ when built with a pre-1.20
toolchain. Official builds of the CLI are already using Go 1.20+ and
should use the new differ without additional changes.
These changes improve CPU time, allocation volume, and the compression
ratio by 57%, 87%, and 52%, respectively.
Benchmark results and analysis:
httpstate ❯ go test -count=10 -run none -benchmem -bench . | tee spanned.txt
httpstate ❯ benchstat buffer-pool.txt spanned.txt >stat.txt
```
│ buffer-pool.txt │ spanned.txt │
│ sec/op │ sec/op vs base │
DiffStack/1_x_2_B-10 45.09µ ± 1% 22.81µ ± 1% -49.41% (p=0.000 n=10)
DiffStack/2_x_2_B-10 79.49µ ± 1% 40.60µ ± 1% -48.93% (p=0.000 n=10)
DiffStack/4_x_2_B-10 172.54µ ± 2% 81.57µ ± 0% -52.72% (p=0.000 n=10)
DiffStack/8_x_2_B-10 455.4µ ± 1% 200.4µ ± 0% -56.00% (p=0.000 n=10)
DiffStack/16_x_2_B-10 1388.2µ ± 1% 575.1µ ± 1% -58.57% (p=0.000 n=10)
DiffStack/32_x_2_B-10 4.409m ± 1% 1.866m ± 0% -57.67% (p=0.000 n=10)
DiffStack/48_x_2_B-10 9.275m ± 0% 3.484m ± 0% -62.44% (p=0.000 n=10)
DiffStack/64_x_2_B-10 14.564m ± 0% 5.945m ± 1% -59.18% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 124.81µ ± 1% 40.14µ ± 1% -67.84% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 299.8µ ± 0% 121.8µ ± 1% -59.38% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 846.2µ ± 1% 355.8µ ± 0% -57.96% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 2.743m ± 1% 1.150m ± 0% -58.06% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 9.271m ± 1% 4.041m ± 0% -56.41% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 31.78m ± 0% 14.97m ± 0% -52.88% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 66.14m ± 1% 30.66m ± 0% -53.63% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 107.50m ± 0% 53.44m ± 0% -50.29% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 240.51µ ± 1% 76.95µ ± 0% -68.01% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 798.5µ ± 0% 310.6µ ± 0% -61.10% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 2451.9µ ± 1% 978.9µ ± 3% -60.08% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 8.258m ± 0% 3.731m ± 0% -54.82% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 28.29m ± 1% 12.77m ± 0% -54.85% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 104.11m ± 1% 44.84m ± 1% -56.93% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 199.73m ± 1% 95.87m ± 1% -52.00% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 339.6m ± 2% 159.2m ± 2% -53.12% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 2721.7µ ± 0% 984.3µ ± 0% -63.84% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 8.334m ± 1% 3.654m ± 2% -56.15% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 25.89m ± 0% 12.23m ± 0% -52.77% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 87.49m ± 1% 40.26m ± 0% -53.98% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 317.0m ± 5% 150.4m ± 0% -52.57% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 681.8m ± 1% 332.1m ± 0% -51.28% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 1190.4m ± 1% 583.1m ± 0% -51.01% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 3.215m ± 0% 1.086m ± 0% -66.21% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 9.676m ± 1% 3.994m ± 0% -58.72% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 29.84m ± 0% 12.88m ± 1% -56.84% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 92.11m ± 1% 42.35m ± 0% -54.02% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 322.1m ± 0% 154.0m ± 0% -52.19% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 54.73m ± 1% 22.63m ± 0% -58.65% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 815.1m ± 0% 395.5m ± 2% -51.48% (p=0.000 n=10)
geomean 10.45m 4.530m -56.65%
│ buffer-pool.txt │ spanned.txt │
│ checkpoint_bytes │ checkpoint_bytes vs base │
DiffStack/1_x_2_B-10 2.539k ± 0% 1.950k ± 0% -23.20% (p=0.000 n=10)
DiffStack/2_x_2_B-10 5.201k ± 0% 4.548k ± 0% -12.56% (p=0.000 n=10)
DiffStack/4_x_2_B-10 13.63k ± 0% 12.30k ± 0% -9.77% (p=0.000 n=10)
DiffStack/8_x_2_B-10 45.09k ± 0% 42.86k ± 0% -4.95% (p=0.000 n=10)
DiffStack/16_x_2_B-10 161.9k ± 0% 156.4k ± 0% -3.42% (p=0.000 n=10)
DiffStack/32_x_2_B-10 579.8k ± 0% 604.2k ± 0% +4.21% (p=0.000 n=10)
DiffStack/48_x_2_B-10 1.306M ± 0% 1.180M ± 0% -9.68% (p=0.000 n=10)
DiffStack/64_x_2_B-10 2.164M ± 0% 2.134M ± 0% -1.36% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 27.109k ± 0% 9.856k ± 0% -63.64% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 78.91k ± 0% 61.59k ± 0% -21.94% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 284.2k ± 0% 258.3k ± 0% -9.13% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 1.119M ± 0% 1.041M ± 0% -6.89% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 4.389M ± 0% 4.201M ± 0% -4.29% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 16.74M ± 0% 17.06M ± 0% +1.95% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 36.30M ± 0% 35.93M ± 0% -1.00% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 60.98M ± 0% 64.06M ± 0% +5.05% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 67.79k ± 0% 34.43k ± 0% -49.20% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 300.1k ± 0% 233.6k ± 0% -22.15% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 1128.3k ± 0% 929.5k ± 0% -17.62% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 4.437M ± 0% 4.238M ± 0% -4.49% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 16.57M ± 0% 15.78M ± 0% -4.79% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 64.35M ± 0% 58.66M ± 0% -8.84% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 127.0M ± 0% 128.0M ± 0% +0.80% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 217.7M ± 0% 216.5M ± 0% -0.54% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 1316.2k ± 0% 921.8k ± 0% -29.97% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 4.733M ± 0% 4.207M ± 0% -11.11% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 16.03M ± 0% 15.77M ± 0% -1.64% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 57.42M ± 0% 54.66M ± 0% -4.81% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 214.8M ± 0% 210.7M ± 0% -1.90% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 472.9M ± 0% 466.9M ± 0% -1.28% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 832.3M ± 0% 824.7M ± 0% -0.92% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 1.575M ± 0% 1.051M ± 0% -33.32% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 5.248M ± 0% 4.199M ± 0% -20.00% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 18.36M ± 0% 15.74M ± 0% -14.28% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 60.86M ± 0% 56.66M ± 0% -6.90% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 223.5M ± 0% 214.0M ± 0% -4.23% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 13.21M ± 0% 13.20M ± 0% -0.01% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 155.9M ± 0% 154.9M ± 0% -0.60% (p=0.000 n=10)
geomean 3.606M 3.173M -12.01%
│ buffer-pool.txt │ spanned.txt │
│ ratio │ ratio vs base │
DiffStack/1_x_2_B-10 789.5m ± 0% 1641.0m ± 0% +107.85% (p=0.000 n=10)
DiffStack/2_x_2_B-10 1.034 ± 0% 2.353 ± 0% +127.56% (p=0.000 n=10)
DiffStack/4_x_2_B-10 1.575 ± 0% 3.592 ± 0% +128.06% (p=0.000 n=10)
DiffStack/8_x_2_B-10 2.829 ± 0% 6.684 ± 0% +136.27% (p=0.000 n=10)
DiffStack/16_x_2_B-10 5.296 ± 0% 12.610 ± 0% +138.10% (p=0.000 n=10)
DiffStack/32_x_2_B-10 9.681 ± 0% 24.790 ± 0% +156.07% (p=0.000 n=10)
DiffStack/48_x_2_B-10 14.65 ± 0% 32.44 ± 0% +121.43% (p=0.000 n=10)
DiffStack/64_x_2_B-10 18.26 ± 0% 44.11 ± 0% +141.57% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 1.383 ± 0% 1.051 ± 0% -24.01% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 1.716 ± 0% 2.324 ± 0% +35.43% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 2.878 ± 0% 4.251 ± 0% +47.71% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 5.475 ± 0% 8.056 ± 0% +47.14% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 10.56 ± 0% 15.77 ± 0% +49.34% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 19.97 ± 0% 31.57 ± 0% +58.09% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 28.80 ± 0% 44.11 ± 0% +53.16% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 36.23 ± 0% 58.84 ± 0% +62.41% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 986.0m ± 0% 1014.0m ± 0% +2.84% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 1.777 ± 0% 2.331 ± 0% +31.18% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 3.057 ± 0% 3.993 ± 0% +30.62% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 5.766 ± 0% 8.511 ± 0% +47.61% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 10.55 ± 0% 15.35 ± 0% +45.50% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 20.28 ± 0% 28.08 ± 0% +38.46% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 26.59 ± 0% 40.64 ± 0% +52.84% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 34.14 ± 0% 51.44 ± 0% +50.67% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 1.993 ± 0% 2.333 ± 0% +17.06% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 3.263 ± 0% 4.569 ± 0% +40.02% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 5.291 ± 0% 7.995 ± 0% +51.11% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 9.275 ± 0% 13.410 ± 0% +44.58% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 17.17 ± 0% 25.44 ± 0% +48.17% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 25.11 ± 0% 37.39 ± 0% +48.90% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 33.09 ± 0% 49.40 ± 0% +49.29% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 1.498 ± 0% 1.999 ± 0% +33.44% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 1.998 ± 0% 2.666 ± 0% +33.43% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 3.180 ± 0% 4.285 ± 0% +34.75% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 5.040 ± 0% 7.199 ± 0% +42.84% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 9.058 ± 0% 13.160 ± 0% +45.29% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 1.997 ± 0% 1.999 ± 0% +0.10% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 36.60 ± 0% 40.42 ± 0% +10.44% (p=0.000 n=10)
geomean 5.993 9.122 +52.20%
│ buffer-pool.txt │ spanned.txt │
│ wire_bytes │ wire_bytes vs base │
DiffStack/1_x_2_B-10 3.216k ± 0% 1.188k ± 0% -63.06% (p=0.000 n=10)
DiffStack/2_x_2_B-10 5.031k ± 0% 1.933k ± 0% -61.58% (p=0.000 n=10)
DiffStack/4_x_2_B-10 8.655k ± 0% 3.424k ± 0% -60.44% (p=0.000 n=10)
DiffStack/8_x_2_B-10 15.938k ± 0% 6.412k ± 0% -59.77% (p=0.000 n=10)
DiffStack/16_x_2_B-10 30.58k ± 0% 12.40k ± 0% -59.45% (p=0.000 n=10)
DiffStack/32_x_2_B-10 59.89k ± 0% 24.37k ± 0% -59.31% (p=0.000 n=10)
DiffStack/48_x_2_B-10 89.17k ± 0% 36.38k ± 0% -59.21% (p=0.000 n=10)
DiffStack/64_x_2_B-10 118.48k ± 0% 48.39k ± 0% -59.16% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 19.596k ± 0% 9.378k ± 0% -52.14% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 45.98k ± 0% 26.50k ± 0% -42.36% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 98.75k ± 0% 60.76k ± 0% -38.47% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 204.3k ± 0% 129.3k ± 0% -36.72% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 415.5k ± 0% 266.3k ± 0% -35.91% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 837.9k ± 0% 540.4k ± 0% -35.50% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 1260.4k ± 0% 814.6k ± 0% -35.37% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 1.683M ± 0% 1.089M ± 0% -35.31% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 68.75k ± 0% 33.95k ± 0% -50.61% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 168.9k ± 0% 100.2k ± 0% -40.64% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 369.1k ± 0% 232.8k ± 0% -36.93% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 769.6k ± 0% 497.9k ± 0% -35.30% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 1.571M ± 0% 1.028M ± 0% -34.53% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 3.173M ± 0% 2.089M ± 0% -34.16% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 4.775M ± 0% 3.149M ± 0% -34.04% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 6.377M ± 0% 4.210M ± 0% -33.98% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 660.4k ± 0% 395.1k ± 0% -40.16% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 1450.4k ± 0% 920.9k ± 0% -36.51% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 3.031M ± 0% 1.972M ± 0% -34.91% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 6.191M ± 0% 4.076M ± 0% -34.17% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 12.512M ± 0% 8.282M ± 0% -33.81% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 18.83M ± 0% 12.49M ± 0% -33.69% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 25.15M ± 0% 16.69M ± 0% -33.63% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 1051.8k ± 0% 525.5k ± 0% -50.04% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 2.626M ± 0% 1.575M ± 0% -40.04% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 5.776M ± 0% 3.673M ± 0% -36.40% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 12.075M ± 0% 7.871M ± 0% -34.82% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 24.67M ± 0% 16.27M ± 0% -34.07% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 6.612M ± 0% 6.606M ± 0% -0.10% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 4.259M ± 0% 3.833M ± 0% -10.00% (p=0.000 n=10)
geomean 601.7k 347.8k -42.19%
│ buffer-pool.txt │ spanned.txt │
│ B/op │ B/op vs base │
DiffStack/1_x_2_B-10 31.85Ki ± 0% 15.79Ki ± 0% -50.42% (p=0.000 n=10)
DiffStack/2_x_2_B-10 61.81Ki ± 0% 28.01Ki ± 0% -54.68% (p=0.000 n=10)
DiffStack/4_x_2_B-10 147.58Ki ± 0% 55.60Ki ± 0% -62.33% (p=0.000 n=10)
DiffStack/8_x_2_B-10 425.9Ki ± 0% 116.9Ki ± 0% -72.55% (p=0.000 n=10)
DiffStack/16_x_2_B-10 1429.9Ki ± 0% 307.8Ki ± 0% -78.47% (p=0.000 n=10)
DiffStack/32_x_2_B-10 4955.7Ki ± 0% 934.1Ki ± 0% -81.15% (p=0.000 n=10)
DiffStack/48_x_2_B-10 10.746Mi ± 0% 1.682Mi ± 0% -84.34% (p=0.000 n=10)
DiffStack/64_x_2_B-10 17.664Mi ± 0% 2.822Mi ± 0% -84.03% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 202.50Ki ± 0% 41.18Ki ± 0% -79.66% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 511.5Ki ± 0% 134.1Ki ± 0% -73.78% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 1513.5Ki ± 0% 309.4Ki ± 0% -79.56% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 4886.0Ki ± 0% 674.2Ki ± 0% -86.20% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 16.287Mi ± 0% 1.427Mi ± 0% -91.24% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 57.541Mi ± 0% 3.198Mi ± 0% -94.44% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 121.703Mi ± 0% 5.810Mi ± 0% -95.23% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 200.588Mi ± 0% 7.495Mi ± 0% -96.26% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 562.5Ki ± 0% 136.3Ki ± 0% -75.78% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 2041.0Ki ± 0% 505.5Ki ± 0% -75.23% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 5.874Mi ± 0% 1.129Mi ± 0% -80.78% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 18.837Mi ± 0% 2.410Mi ± 0% -87.21% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 61.898Mi ± 0% 5.015Mi ± 0% -91.90% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 221.69Mi ± 0% 10.36Mi ± 0% -95.32% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 430.13Mi ± 0% 18.50Mi ± 0% -95.70% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 722.07Mi ± 0% 21.63Mi ± 0% -97.00% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 8.098Mi ± 0% 1.656Mi ± 0% -79.55% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 22.662Mi ± 0% 3.934Mi ± 0% -82.64% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 65.217Mi ± 0% 8.425Mi ± 0% -87.08% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 204.90Mi ± 0% 17.22Mi ± 1% -91.60% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 703.12Mi ± 0% 34.89Mi ± 0% -95.04% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 1505.36Mi ± 0% 61.24Mi ± 0% -95.93% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 2586.04Mi ± 0% 70.48Mi ± 1% -97.27% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 11.854Mi ± 1% 2.127Mi ± 0% -82.06% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 32.932Mi ± 1% 6.558Mi ± 1% -80.09% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 90.04Mi ± 0% 15.15Mi ± 0% -83.17% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 252.94Mi ± 0% 32.27Mi ± 1% -87.24% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 801.77Mi ± 0% 66.46Mi ± 1% -91.71% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 103.67Mi ± 0% 35.52Mi ± 0% -65.74% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 913.90Mi ± 0% 53.53Mi ± 1% -94.14% (p=0.000 n=10)
geomean 19.16Mi 2.409Mi -87.43%
│ buffer-pool.txt │ spanned.txt │
│ allocs/op │ allocs/op vs base │
DiffStack/1_x_2_B-10 344.0 ± 0% 184.0 ± 0% -46.51% (p=0.000 n=10)
DiffStack/2_x_2_B-10 616.0 ± 0% 295.0 ± 0% -52.11% (p=0.000 n=10)
DiffStack/4_x_2_B-10 1381.0 ± 0% 524.0 ± 0% -62.06% (p=0.000 n=10)
DiffStack/8_x_2_B-10 3.922k ± 0% 1.032k ± 0% -73.69% (p=0.000 n=10)
DiffStack/16_x_2_B-10 12.791k ± 0% 2.233k ± 0% -82.54% (p=0.000 n=10)
DiffStack/32_x_2_B-10 43.416k ± 0% 5.410k ± 0% -87.54% (p=0.000 n=10)
DiffStack/48_x_2_B-10 95.803k ± 0% 9.036k ± 0% -90.57% (p=0.000 n=10)
DiffStack/64_x_2_B-10 156.99k ± 0% 13.99k ± 0% -91.09% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 335.0 ± 0% 182.0 ± 0% -45.67% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 597.5 ± 0% 295.0 ± 0% -50.63% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 1304.0 ± 0% 527.0 ± 0% -59.59% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 3.475k ± 0% 1.036k ± 0% -70.19% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 10.677k ± 0% 2.239k ± 0% -79.03% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 35.357k ± 0% 5.414k ± 0% -84.69% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 72.782k ± 0% 9.296k ± 0% -87.23% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 119.24k ± 0% 14.26k ± 0% -88.04% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 309.0 ± 0% 182.0 ± 0% -41.10% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 594.0 ± 0% 296.0 ± 0% -50.17% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 1324.5 ± 0% 528.0 ± 0% -60.14% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 3.554k ± 0% 1.047k ± 0% -70.54% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 10.511k ± 0% 2.232k ± 0% -78.77% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 35.147k ± 0% 5.194k ± 0% -85.22% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 66.519k ± 0% 8.958k ± 0% -86.53% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 110.71k ± 0% 13.28k ± 0% -88.00% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 644.0 ± 0% 296.0 ± 0% -54.04% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 1418.5 ± 0% 535.0 ± 0% -62.28% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 3.497k ± 0% 1.047k ± 0% -70.06% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 9.805k ± 0% 2.182k ± 0% -77.75% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 30.996k ± 0% 5.040k ± 0% -83.74% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 63.751k ± 0% 8.656k ± 0% -86.42% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 107.97k ± 0% 13.03k ± 0% -87.93% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 356.0 ± 0% 186.0 ± 0% -47.75% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 671.0 ± 0% 300.0 ± 0% -55.29% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 1467.5 ± 0% 536.0 ± 0% -63.48% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 3.553k ± 0% 1.038k ± 0% -70.80% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 9.971k ± 0% 2.182k ± 0% -78.12% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 489.16k ± 0% 73.65k ± 0% -84.94% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 7236.9k ± 0% 857.8k ± 0% -88.15% (p=0.000 n=10)
geomean 7.989k 1.934k -75.79%
```
2023-05-05 02:15:09 +00:00
|
|
|
//go:build !go1.20
|
|
|
|
|
2022-09-19 16:44:37 +00:00
|
|
|
// Copyright 2016-2022, Pulumi Corporation.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package client
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bytes"
|
|
|
|
"encoding/json"
|
2022-12-06 21:19:23 +00:00
|
|
|
"io"
|
|
|
|
"math"
|
|
|
|
|
2023-05-05 00:12:34 +00:00
|
|
|
jsoniter "github.com/json-iterator/go"
|
2022-09-19 16:44:37 +00:00
|
|
|
|
|
|
|
"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
|
|
|
|
)
|
|
|
|
|
2022-12-15 17:12:32 +00:00
|
|
|
const maxNewLines = 1024
|
|
|
|
|
2022-12-06 21:19:23 +00:00
|
|
|
var jsonIterConfig = jsoniter.Config{SortMapKeys: true}.Froze()
|
|
|
|
|
|
|
|
// Marshals to canonical JSON in the apitype.UntypedDeployment format.
|
|
|
|
//
|
|
|
|
// Optimized for large checkpoints.
|
|
|
|
//
|
|
|
|
// Injects newlines to allow efficient textual diffs over the JSON. Textual diffs currently use O(N^2) memory in the
|
2022-12-15 17:12:32 +00:00
|
|
|
// number of newlines, so the injection needs to be conservative. Currently it limits to up to maxNewLines newlines
|
|
|
|
// which would result in max 8MB memory use by the algorithm.
|
2023-05-05 00:12:34 +00:00
|
|
|
func MarshalUntypedDeployment(b *bytes.Buffer, deployment *apitype.DeploymentV3) error {
|
2023-03-03 16:36:39 +00:00
|
|
|
md := &marshalUntypedDeployment{deployment}
|
2023-05-05 00:12:34 +00:00
|
|
|
return md.Write(b)
|
2022-12-06 21:19:23 +00:00
|
|
|
}
|
|
|
|
|
[snapshot] Use a newer diff package
The version of gopls that gotextdiff is derived from is over three years
old. In the intervening time, the differ used by gopls has improved
tremendously, and the language has grown new features like generics.
These changes use a different diff package that is derived from an
up-to-date version of gopls and exposes an API that is parameterized
over the storage of the text. Taken together, this allows us to use a
much faster and more memory-efficient algorithm for diffing while
safely avoiding string copies when invoking the differ.
The more efficient algorithm operates on arbitrary slices whose elements
are comparable. Naive use of this algorithm--passing byte slices
in--can cause increased time spent diffing, as the sequences being
compared are longer. In order to avoid this, these changes record
spans of marshaled deployments as the deployments are marshaled. These
spans are strings that cover the diffable elements of the deployment:
its header, its resources, and its pending operations.
The output of the newer algorithm is converible to the format we
currently send to the service, so these changes should not require
service-side changes.
In order to avoid bumping the minimum Go version required to build the
`github.com/pulumi/pulumi/pkg/v3" and therefore break backwards
compatibility with consumers using older toolchains (e.g. providers),
these changes use the existing differ when built with a pre-1.20
toolchain. Official builds of the CLI are already using Go 1.20+ and
should use the new differ without additional changes.
These changes improve CPU time, allocation volume, and the compression
ratio by 57%, 87%, and 52%, respectively.
Benchmark results and analysis:
httpstate ❯ go test -count=10 -run none -benchmem -bench . | tee spanned.txt
httpstate ❯ benchstat buffer-pool.txt spanned.txt >stat.txt
```
│ buffer-pool.txt │ spanned.txt │
│ sec/op │ sec/op vs base │
DiffStack/1_x_2_B-10 45.09µ ± 1% 22.81µ ± 1% -49.41% (p=0.000 n=10)
DiffStack/2_x_2_B-10 79.49µ ± 1% 40.60µ ± 1% -48.93% (p=0.000 n=10)
DiffStack/4_x_2_B-10 172.54µ ± 2% 81.57µ ± 0% -52.72% (p=0.000 n=10)
DiffStack/8_x_2_B-10 455.4µ ± 1% 200.4µ ± 0% -56.00% (p=0.000 n=10)
DiffStack/16_x_2_B-10 1388.2µ ± 1% 575.1µ ± 1% -58.57% (p=0.000 n=10)
DiffStack/32_x_2_B-10 4.409m ± 1% 1.866m ± 0% -57.67% (p=0.000 n=10)
DiffStack/48_x_2_B-10 9.275m ± 0% 3.484m ± 0% -62.44% (p=0.000 n=10)
DiffStack/64_x_2_B-10 14.564m ± 0% 5.945m ± 1% -59.18% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 124.81µ ± 1% 40.14µ ± 1% -67.84% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 299.8µ ± 0% 121.8µ ± 1% -59.38% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 846.2µ ± 1% 355.8µ ± 0% -57.96% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 2.743m ± 1% 1.150m ± 0% -58.06% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 9.271m ± 1% 4.041m ± 0% -56.41% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 31.78m ± 0% 14.97m ± 0% -52.88% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 66.14m ± 1% 30.66m ± 0% -53.63% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 107.50m ± 0% 53.44m ± 0% -50.29% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 240.51µ ± 1% 76.95µ ± 0% -68.01% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 798.5µ ± 0% 310.6µ ± 0% -61.10% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 2451.9µ ± 1% 978.9µ ± 3% -60.08% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 8.258m ± 0% 3.731m ± 0% -54.82% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 28.29m ± 1% 12.77m ± 0% -54.85% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 104.11m ± 1% 44.84m ± 1% -56.93% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 199.73m ± 1% 95.87m ± 1% -52.00% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 339.6m ± 2% 159.2m ± 2% -53.12% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 2721.7µ ± 0% 984.3µ ± 0% -63.84% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 8.334m ± 1% 3.654m ± 2% -56.15% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 25.89m ± 0% 12.23m ± 0% -52.77% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 87.49m ± 1% 40.26m ± 0% -53.98% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 317.0m ± 5% 150.4m ± 0% -52.57% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 681.8m ± 1% 332.1m ± 0% -51.28% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 1190.4m ± 1% 583.1m ± 0% -51.01% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 3.215m ± 0% 1.086m ± 0% -66.21% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 9.676m ± 1% 3.994m ± 0% -58.72% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 29.84m ± 0% 12.88m ± 1% -56.84% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 92.11m ± 1% 42.35m ± 0% -54.02% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 322.1m ± 0% 154.0m ± 0% -52.19% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 54.73m ± 1% 22.63m ± 0% -58.65% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 815.1m ± 0% 395.5m ± 2% -51.48% (p=0.000 n=10)
geomean 10.45m 4.530m -56.65%
│ buffer-pool.txt │ spanned.txt │
│ checkpoint_bytes │ checkpoint_bytes vs base │
DiffStack/1_x_2_B-10 2.539k ± 0% 1.950k ± 0% -23.20% (p=0.000 n=10)
DiffStack/2_x_2_B-10 5.201k ± 0% 4.548k ± 0% -12.56% (p=0.000 n=10)
DiffStack/4_x_2_B-10 13.63k ± 0% 12.30k ± 0% -9.77% (p=0.000 n=10)
DiffStack/8_x_2_B-10 45.09k ± 0% 42.86k ± 0% -4.95% (p=0.000 n=10)
DiffStack/16_x_2_B-10 161.9k ± 0% 156.4k ± 0% -3.42% (p=0.000 n=10)
DiffStack/32_x_2_B-10 579.8k ± 0% 604.2k ± 0% +4.21% (p=0.000 n=10)
DiffStack/48_x_2_B-10 1.306M ± 0% 1.180M ± 0% -9.68% (p=0.000 n=10)
DiffStack/64_x_2_B-10 2.164M ± 0% 2.134M ± 0% -1.36% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 27.109k ± 0% 9.856k ± 0% -63.64% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 78.91k ± 0% 61.59k ± 0% -21.94% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 284.2k ± 0% 258.3k ± 0% -9.13% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 1.119M ± 0% 1.041M ± 0% -6.89% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 4.389M ± 0% 4.201M ± 0% -4.29% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 16.74M ± 0% 17.06M ± 0% +1.95% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 36.30M ± 0% 35.93M ± 0% -1.00% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 60.98M ± 0% 64.06M ± 0% +5.05% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 67.79k ± 0% 34.43k ± 0% -49.20% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 300.1k ± 0% 233.6k ± 0% -22.15% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 1128.3k ± 0% 929.5k ± 0% -17.62% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 4.437M ± 0% 4.238M ± 0% -4.49% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 16.57M ± 0% 15.78M ± 0% -4.79% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 64.35M ± 0% 58.66M ± 0% -8.84% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 127.0M ± 0% 128.0M ± 0% +0.80% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 217.7M ± 0% 216.5M ± 0% -0.54% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 1316.2k ± 0% 921.8k ± 0% -29.97% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 4.733M ± 0% 4.207M ± 0% -11.11% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 16.03M ± 0% 15.77M ± 0% -1.64% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 57.42M ± 0% 54.66M ± 0% -4.81% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 214.8M ± 0% 210.7M ± 0% -1.90% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 472.9M ± 0% 466.9M ± 0% -1.28% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 832.3M ± 0% 824.7M ± 0% -0.92% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 1.575M ± 0% 1.051M ± 0% -33.32% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 5.248M ± 0% 4.199M ± 0% -20.00% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 18.36M ± 0% 15.74M ± 0% -14.28% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 60.86M ± 0% 56.66M ± 0% -6.90% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 223.5M ± 0% 214.0M ± 0% -4.23% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 13.21M ± 0% 13.20M ± 0% -0.01% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 155.9M ± 0% 154.9M ± 0% -0.60% (p=0.000 n=10)
geomean 3.606M 3.173M -12.01%
│ buffer-pool.txt │ spanned.txt │
│ ratio │ ratio vs base │
DiffStack/1_x_2_B-10 789.5m ± 0% 1641.0m ± 0% +107.85% (p=0.000 n=10)
DiffStack/2_x_2_B-10 1.034 ± 0% 2.353 ± 0% +127.56% (p=0.000 n=10)
DiffStack/4_x_2_B-10 1.575 ± 0% 3.592 ± 0% +128.06% (p=0.000 n=10)
DiffStack/8_x_2_B-10 2.829 ± 0% 6.684 ± 0% +136.27% (p=0.000 n=10)
DiffStack/16_x_2_B-10 5.296 ± 0% 12.610 ± 0% +138.10% (p=0.000 n=10)
DiffStack/32_x_2_B-10 9.681 ± 0% 24.790 ± 0% +156.07% (p=0.000 n=10)
DiffStack/48_x_2_B-10 14.65 ± 0% 32.44 ± 0% +121.43% (p=0.000 n=10)
DiffStack/64_x_2_B-10 18.26 ± 0% 44.11 ± 0% +141.57% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 1.383 ± 0% 1.051 ± 0% -24.01% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 1.716 ± 0% 2.324 ± 0% +35.43% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 2.878 ± 0% 4.251 ± 0% +47.71% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 5.475 ± 0% 8.056 ± 0% +47.14% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 10.56 ± 0% 15.77 ± 0% +49.34% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 19.97 ± 0% 31.57 ± 0% +58.09% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 28.80 ± 0% 44.11 ± 0% +53.16% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 36.23 ± 0% 58.84 ± 0% +62.41% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 986.0m ± 0% 1014.0m ± 0% +2.84% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 1.777 ± 0% 2.331 ± 0% +31.18% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 3.057 ± 0% 3.993 ± 0% +30.62% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 5.766 ± 0% 8.511 ± 0% +47.61% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 10.55 ± 0% 15.35 ± 0% +45.50% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 20.28 ± 0% 28.08 ± 0% +38.46% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 26.59 ± 0% 40.64 ± 0% +52.84% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 34.14 ± 0% 51.44 ± 0% +50.67% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 1.993 ± 0% 2.333 ± 0% +17.06% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 3.263 ± 0% 4.569 ± 0% +40.02% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 5.291 ± 0% 7.995 ± 0% +51.11% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 9.275 ± 0% 13.410 ± 0% +44.58% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 17.17 ± 0% 25.44 ± 0% +48.17% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 25.11 ± 0% 37.39 ± 0% +48.90% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 33.09 ± 0% 49.40 ± 0% +49.29% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 1.498 ± 0% 1.999 ± 0% +33.44% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 1.998 ± 0% 2.666 ± 0% +33.43% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 3.180 ± 0% 4.285 ± 0% +34.75% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 5.040 ± 0% 7.199 ± 0% +42.84% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 9.058 ± 0% 13.160 ± 0% +45.29% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 1.997 ± 0% 1.999 ± 0% +0.10% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 36.60 ± 0% 40.42 ± 0% +10.44% (p=0.000 n=10)
geomean 5.993 9.122 +52.20%
│ buffer-pool.txt │ spanned.txt │
│ wire_bytes │ wire_bytes vs base │
DiffStack/1_x_2_B-10 3.216k ± 0% 1.188k ± 0% -63.06% (p=0.000 n=10)
DiffStack/2_x_2_B-10 5.031k ± 0% 1.933k ± 0% -61.58% (p=0.000 n=10)
DiffStack/4_x_2_B-10 8.655k ± 0% 3.424k ± 0% -60.44% (p=0.000 n=10)
DiffStack/8_x_2_B-10 15.938k ± 0% 6.412k ± 0% -59.77% (p=0.000 n=10)
DiffStack/16_x_2_B-10 30.58k ± 0% 12.40k ± 0% -59.45% (p=0.000 n=10)
DiffStack/32_x_2_B-10 59.89k ± 0% 24.37k ± 0% -59.31% (p=0.000 n=10)
DiffStack/48_x_2_B-10 89.17k ± 0% 36.38k ± 0% -59.21% (p=0.000 n=10)
DiffStack/64_x_2_B-10 118.48k ± 0% 48.39k ± 0% -59.16% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 19.596k ± 0% 9.378k ± 0% -52.14% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 45.98k ± 0% 26.50k ± 0% -42.36% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 98.75k ± 0% 60.76k ± 0% -38.47% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 204.3k ± 0% 129.3k ± 0% -36.72% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 415.5k ± 0% 266.3k ± 0% -35.91% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 837.9k ± 0% 540.4k ± 0% -35.50% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 1260.4k ± 0% 814.6k ± 0% -35.37% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 1.683M ± 0% 1.089M ± 0% -35.31% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 68.75k ± 0% 33.95k ± 0% -50.61% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 168.9k ± 0% 100.2k ± 0% -40.64% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 369.1k ± 0% 232.8k ± 0% -36.93% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 769.6k ± 0% 497.9k ± 0% -35.30% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 1.571M ± 0% 1.028M ± 0% -34.53% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 3.173M ± 0% 2.089M ± 0% -34.16% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 4.775M ± 0% 3.149M ± 0% -34.04% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 6.377M ± 0% 4.210M ± 0% -33.98% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 660.4k ± 0% 395.1k ± 0% -40.16% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 1450.4k ± 0% 920.9k ± 0% -36.51% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 3.031M ± 0% 1.972M ± 0% -34.91% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 6.191M ± 0% 4.076M ± 0% -34.17% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 12.512M ± 0% 8.282M ± 0% -33.81% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 18.83M ± 0% 12.49M ± 0% -33.69% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 25.15M ± 0% 16.69M ± 0% -33.63% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 1051.8k ± 0% 525.5k ± 0% -50.04% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 2.626M ± 0% 1.575M ± 0% -40.04% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 5.776M ± 0% 3.673M ± 0% -36.40% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 12.075M ± 0% 7.871M ± 0% -34.82% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 24.67M ± 0% 16.27M ± 0% -34.07% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 6.612M ± 0% 6.606M ± 0% -0.10% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 4.259M ± 0% 3.833M ± 0% -10.00% (p=0.000 n=10)
geomean 601.7k 347.8k -42.19%
│ buffer-pool.txt │ spanned.txt │
│ B/op │ B/op vs base │
DiffStack/1_x_2_B-10 31.85Ki ± 0% 15.79Ki ± 0% -50.42% (p=0.000 n=10)
DiffStack/2_x_2_B-10 61.81Ki ± 0% 28.01Ki ± 0% -54.68% (p=0.000 n=10)
DiffStack/4_x_2_B-10 147.58Ki ± 0% 55.60Ki ± 0% -62.33% (p=0.000 n=10)
DiffStack/8_x_2_B-10 425.9Ki ± 0% 116.9Ki ± 0% -72.55% (p=0.000 n=10)
DiffStack/16_x_2_B-10 1429.9Ki ± 0% 307.8Ki ± 0% -78.47% (p=0.000 n=10)
DiffStack/32_x_2_B-10 4955.7Ki ± 0% 934.1Ki ± 0% -81.15% (p=0.000 n=10)
DiffStack/48_x_2_B-10 10.746Mi ± 0% 1.682Mi ± 0% -84.34% (p=0.000 n=10)
DiffStack/64_x_2_B-10 17.664Mi ± 0% 2.822Mi ± 0% -84.03% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 202.50Ki ± 0% 41.18Ki ± 0% -79.66% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 511.5Ki ± 0% 134.1Ki ± 0% -73.78% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 1513.5Ki ± 0% 309.4Ki ± 0% -79.56% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 4886.0Ki ± 0% 674.2Ki ± 0% -86.20% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 16.287Mi ± 0% 1.427Mi ± 0% -91.24% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 57.541Mi ± 0% 3.198Mi ± 0% -94.44% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 121.703Mi ± 0% 5.810Mi ± 0% -95.23% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 200.588Mi ± 0% 7.495Mi ± 0% -96.26% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 562.5Ki ± 0% 136.3Ki ± 0% -75.78% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 2041.0Ki ± 0% 505.5Ki ± 0% -75.23% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 5.874Mi ± 0% 1.129Mi ± 0% -80.78% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 18.837Mi ± 0% 2.410Mi ± 0% -87.21% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 61.898Mi ± 0% 5.015Mi ± 0% -91.90% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 221.69Mi ± 0% 10.36Mi ± 0% -95.32% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 430.13Mi ± 0% 18.50Mi ± 0% -95.70% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 722.07Mi ± 0% 21.63Mi ± 0% -97.00% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 8.098Mi ± 0% 1.656Mi ± 0% -79.55% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 22.662Mi ± 0% 3.934Mi ± 0% -82.64% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 65.217Mi ± 0% 8.425Mi ± 0% -87.08% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 204.90Mi ± 0% 17.22Mi ± 1% -91.60% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 703.12Mi ± 0% 34.89Mi ± 0% -95.04% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 1505.36Mi ± 0% 61.24Mi ± 0% -95.93% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 2586.04Mi ± 0% 70.48Mi ± 1% -97.27% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 11.854Mi ± 1% 2.127Mi ± 0% -82.06% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 32.932Mi ± 1% 6.558Mi ± 1% -80.09% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 90.04Mi ± 0% 15.15Mi ± 0% -83.17% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 252.94Mi ± 0% 32.27Mi ± 1% -87.24% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 801.77Mi ± 0% 66.46Mi ± 1% -91.71% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 103.67Mi ± 0% 35.52Mi ± 0% -65.74% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 913.90Mi ± 0% 53.53Mi ± 1% -94.14% (p=0.000 n=10)
geomean 19.16Mi 2.409Mi -87.43%
│ buffer-pool.txt │ spanned.txt │
│ allocs/op │ allocs/op vs base │
DiffStack/1_x_2_B-10 344.0 ± 0% 184.0 ± 0% -46.51% (p=0.000 n=10)
DiffStack/2_x_2_B-10 616.0 ± 0% 295.0 ± 0% -52.11% (p=0.000 n=10)
DiffStack/4_x_2_B-10 1381.0 ± 0% 524.0 ± 0% -62.06% (p=0.000 n=10)
DiffStack/8_x_2_B-10 3.922k ± 0% 1.032k ± 0% -73.69% (p=0.000 n=10)
DiffStack/16_x_2_B-10 12.791k ± 0% 2.233k ± 0% -82.54% (p=0.000 n=10)
DiffStack/32_x_2_B-10 43.416k ± 0% 5.410k ± 0% -87.54% (p=0.000 n=10)
DiffStack/48_x_2_B-10 95.803k ± 0% 9.036k ± 0% -90.57% (p=0.000 n=10)
DiffStack/64_x_2_B-10 156.99k ± 0% 13.99k ± 0% -91.09% (p=0.000 n=10)
DiffStack/1_x_8.2_kB-10 335.0 ± 0% 182.0 ± 0% -45.67% (p=0.000 n=10)
DiffStack/2_x_8.2_kB-10 597.5 ± 0% 295.0 ± 0% -50.63% (p=0.000 n=10)
DiffStack/4_x_8.2_kB-10 1304.0 ± 0% 527.0 ± 0% -59.59% (p=0.000 n=10)
DiffStack/8_x_8.2_kB-10 3.475k ± 0% 1.036k ± 0% -70.19% (p=0.000 n=10)
DiffStack/16_x_8.2_kB-10 10.677k ± 0% 2.239k ± 0% -79.03% (p=0.000 n=10)
DiffStack/32_x_8.2_kB-10 35.357k ± 0% 5.414k ± 0% -84.69% (p=0.000 n=10)
DiffStack/48_x_8.2_kB-10 72.782k ± 0% 9.296k ± 0% -87.23% (p=0.000 n=10)
DiffStack/64_x_8.2_kB-10 119.24k ± 0% 14.26k ± 0% -88.04% (p=0.000 n=10)
DiffStack/1_x_33_kB-10 309.0 ± 0% 182.0 ± 0% -41.10% (p=0.000 n=10)
DiffStack/2_x_33_kB-10 594.0 ± 0% 296.0 ± 0% -50.17% (p=0.000 n=10)
DiffStack/4_x_33_kB-10 1324.5 ± 0% 528.0 ± 0% -60.14% (p=0.000 n=10)
DiffStack/8_x_33_kB-10 3.554k ± 0% 1.047k ± 0% -70.54% (p=0.000 n=10)
DiffStack/16_x_33_kB-10 10.511k ± 0% 2.232k ± 0% -78.77% (p=0.000 n=10)
DiffStack/32_x_33_kB-10 35.147k ± 0% 5.194k ± 0% -85.22% (p=0.000 n=10)
DiffStack/48_x_33_kB-10 66.519k ± 0% 8.958k ± 0% -86.53% (p=0.000 n=10)
DiffStack/64_x_33_kB-10 110.71k ± 0% 13.28k ± 0% -88.00% (p=0.000 n=10)
DiffStack/2_x_131_kB-10 644.0 ± 0% 296.0 ± 0% -54.04% (p=0.000 n=10)
DiffStack/4_x_131_kB-10 1418.5 ± 0% 535.0 ± 0% -62.28% (p=0.000 n=10)
DiffStack/8_x_131_kB-10 3.497k ± 0% 1.047k ± 0% -70.06% (p=0.000 n=10)
DiffStack/16_x_131_kB-10 9.805k ± 0% 2.182k ± 0% -77.75% (p=0.000 n=10)
DiffStack/32_x_131_kB-10 30.996k ± 0% 5.040k ± 0% -83.74% (p=0.000 n=10)
DiffStack/48_x_131_kB-10 63.751k ± 0% 8.656k ± 0% -86.42% (p=0.000 n=10)
DiffStack/64_x_131_kB-10 107.97k ± 0% 13.03k ± 0% -87.93% (p=0.000 n=10)
DiffStack/1_x_524_kB-10 356.0 ± 0% 186.0 ± 0% -47.75% (p=0.000 n=10)
DiffStack/2_x_524_kB-10 671.0 ± 0% 300.0 ± 0% -55.29% (p=0.000 n=10)
DiffStack/4_x_524_kB-10 1467.5 ± 0% 536.0 ± 0% -63.48% (p=0.000 n=10)
DiffStack/8_x_524_kB-10 3.553k ± 0% 1.038k ± 0% -70.80% (p=0.000 n=10)
DiffStack/16_x_524_kB-10 9.971k ± 0% 2.182k ± 0% -78.12% (p=0.000 n=10)
DiffStackRecorded/two-large-checkpoints.json-10 489.16k ± 0% 73.65k ± 0% -84.94% (p=0.000 n=10)
DiffStackRecorded/checkpoints.json-10 7236.9k ± 0% 857.8k ± 0% -88.15% (p=0.000 n=10)
geomean 7.989k 1.934k -75.79%
```
2023-05-05 02:15:09 +00:00
|
|
|
func marshalDeployment(d *apitype.DeploymentV3) (json.RawMessage, error) {
|
|
|
|
var b bytes.Buffer
|
|
|
|
if err := MarshalUntypedDeployment(&b, d); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return json.RawMessage(b.Bytes()), nil
|
|
|
|
}
|
|
|
|
|
2022-12-06 21:19:23 +00:00
|
|
|
func marshalVerbatimCheckpointRequest(req apitype.PatchUpdateVerbatimCheckpointRequest) (json.RawMessage, error) {
|
|
|
|
// Unlike encoding/json, using jsonIter here will not reindent req.UntypedDeployment, which is what is needed
|
|
|
|
// for the Verbatim protocol.
|
|
|
|
return jsonIterConfig.Marshal(req)
|
|
|
|
}
|
|
|
|
|
|
|
|
type marshalUntypedDeployment struct {
|
|
|
|
deployment *apitype.DeploymentV3
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *marshalUntypedDeployment) Write(w io.Writer) error {
|
|
|
|
cfg := jsonIterConfig
|
|
|
|
stream := cfg.BorrowStream(w)
|
|
|
|
defer cfg.ReturnStream(stream)
|
|
|
|
err := c.writeToStream(stream)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *marshalUntypedDeployment) writeToStream(stream *jsoniter.Stream) error {
|
2022-12-15 17:12:32 +00:00
|
|
|
stream.WriteObjectStart() // writes `{`
|
|
|
|
stream.WriteObjectField("version") // writes `"version":`
|
2022-12-06 21:19:23 +00:00
|
|
|
stream.WriteInt(3)
|
2022-12-15 17:12:32 +00:00
|
|
|
stream.WriteMore() // writes `,`
|
2022-12-06 21:19:23 +00:00
|
|
|
stream.WriteObjectField("deployment")
|
2022-12-15 17:14:51 +00:00
|
|
|
err := c.writeDeploymentV3(stream)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2022-12-15 17:12:32 +00:00
|
|
|
stream.WriteObjectEnd() // writes `}`
|
2022-12-06 21:19:23 +00:00
|
|
|
return stream.Flush()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (c *marshalUntypedDeployment) writeDeploymentV3(stream *jsoniter.Stream) (err error) {
|
|
|
|
deployment := c.deployment
|
|
|
|
stream.WriteObjectStart()
|
|
|
|
stream.WriteObjectField("manifest")
|
|
|
|
stream.WriteVal(deployment.Manifest)
|
|
|
|
if deployment.SecretsProviders != nil {
|
|
|
|
stream.WriteMore()
|
|
|
|
stream.WriteObjectField("secrets_providers")
|
|
|
|
stream.WriteVal(deployment.SecretsProviders)
|
|
|
|
}
|
|
|
|
if err = stream.Flush(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
nResources := len(deployment.Resources)
|
|
|
|
|
2022-12-15 17:12:32 +00:00
|
|
|
maxNL := maxNewLines - 2
|
|
|
|
newlinePeriod := int(math.Ceil(float64(nResources) / float64(maxNL)))
|
2022-12-06 21:19:23 +00:00
|
|
|
|
|
|
|
if nResources > 0 {
|
|
|
|
stream.WriteMore()
|
|
|
|
stream.WriteObjectField("resources")
|
|
|
|
stream.WriteRaw("[\n")
|
|
|
|
for i, r := range deployment.Resources {
|
|
|
|
if i > 0 {
|
|
|
|
stream.WriteRaw(",")
|
2022-12-15 17:12:32 +00:00
|
|
|
if (nResources <= maxNL) || (i%newlinePeriod == 0) {
|
2022-12-06 21:19:23 +00:00
|
|
|
stream.WriteRaw("\n")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
stream.WriteVal(r)
|
|
|
|
if err = stream.Flush(); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
stream.WriteRaw("\n]")
|
|
|
|
}
|
|
|
|
if len(deployment.PendingOperations) > 0 {
|
|
|
|
stream.WriteMore()
|
|
|
|
stream.WriteObjectField("pendingOperations")
|
|
|
|
stream.WriteVal(deployment.PendingOperations)
|
|
|
|
}
|
|
|
|
stream.WriteObjectEnd()
|
|
|
|
return stream.Flush()
|
2022-09-19 16:44:37 +00:00
|
|
|
}
|