2022-09-01 18:39:09 +00:00
|
|
|
// Copyright 2016-2022, Pulumi Corporation.
|
2018-05-22 19:43:36 +00:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
Add Promises leak and hang detection
We have an issue in the runtime right now where we serialize closures
asynchronously, meaning we make it possible to form cycles between
resource graphs (something that ought to be impossible in our model,
where resources are "immutable" after creation and cannot form cycles).
Let me tell you a tale of debugging this ...
Well, no, let's not do that. But thankfully I've left behind some
little utilities that might make debugging such a thing easier down
the road. Namely:
* By default, most of our core runtime promises leverage a leak handler
that will log an error message should the process exit with certain
critical unresolved promises. This error message will include some
handy context (like whether it was an input promise) as well as a
stack trace for its point of creation.
* Optionally, with a flag in runtime/debuggable.ts, you may wire up
a hang detector, for situations where we may want to detect this
situation sooner than process exit, using the regular message loop.
This uses a defined timeout, prints the same diagnostics as the
leak detector when a hang is detected, and is disabled by default.
2017-09-07 01:35:20 +00:00
|
|
|
|
2017-10-08 19:10:46 +00:00
|
|
|
import * as log from "../log";
|
2022-09-01 18:39:09 +00:00
|
|
|
import * as state from "./state";
|
2023-06-15 14:43:05 +00:00
|
|
|
/** @internal
|
2017-09-22 01:15:29 +00:00
|
|
|
* debugPromiseLeaks can be set to enable promises leaks debugging.
|
|
|
|
*/
|
2023-06-15 14:43:05 +00:00
|
|
|
export const debugPromiseLeaks: boolean = !!process.env.PULUMI_DEBUG_PROMISE_LEAKS;
|
Eliminate Computed/Property in favor of Promises
As part of pulumi/pulumi-fabric#331, we've been exploring just using
undefined to indicate that a property value is absent during planning.
We also considered blocking the message loop to simplify the overall
programming model, so that all asynchrony is hidden.
It turns out ThereBeDragons :dragon_face: anytime you try to block the
message loop. So, we aren't quite sure about that bit.
But the part we are convicted about is that this Computed/Property
model is far too complex. Furthermore, it's very close to promises, and
yet frustratingly so far away. Indeed, the original thinking in
pulumi/pulumi-fabric#271 was simply to use promises, but we wanted to
encourage dataflow styles, rather than control flow. But we muddied up
our thinking by worrying about awaiting a promise that would never resolve.
It turns out we can achieve a middle ground: resolve planning promises to
undefined, so that they don't lead to hangs, but still use promises so
that asynchrony is explicit in the system. This also avoids blocking the
message loop. Who knows, this may actually be a fine final destination.
2017-09-20 14:40:09 +00:00
|
|
|
|
2017-09-22 01:15:29 +00:00
|
|
|
/**
|
|
|
|
* leakDetectorScheduled is true when the promise leak detector is scheduled for process exit.
|
|
|
|
*/
|
Add Promises leak and hang detection
We have an issue in the runtime right now where we serialize closures
asynchronously, meaning we make it possible to form cycles between
resource graphs (something that ought to be impossible in our model,
where resources are "immutable" after creation and cannot form cycles).
Let me tell you a tale of debugging this ...
Well, no, let's not do that. But thankfully I've left behind some
little utilities that might make debugging such a thing easier down
the road. Namely:
* By default, most of our core runtime promises leverage a leak handler
that will log an error message should the process exit with certain
critical unresolved promises. This error message will include some
handy context (like whether it was an input promise) as well as a
stack trace for its point of creation.
* Optionally, with a flag in runtime/debuggable.ts, you may wire up
a hang detector, for situations where we may want to detect this
situation sooner than process exit, using the regular message loop.
This uses a defined timeout, prints the same diagnostics as the
leak detector when a hang is detected, and is disabled by default.
2017-09-07 01:35:20 +00:00
|
|
|
let leakDetectorScheduled: boolean = false;
|
2020-09-24 02:06:26 +00:00
|
|
|
|
2022-09-20 18:54:06 +00:00
|
|
|
/** @internal */
|
2020-09-24 02:06:26 +00:00
|
|
|
export function leakedPromises(): [Set<Promise<any>>, string] {
|
2022-09-01 18:39:09 +00:00
|
|
|
const localStore = state.getStore();
|
|
|
|
const leaked = localStore.leakCandidates;
|
|
|
|
const promisePlural = leaked.size === 1 ? "promise was" : "promises were";
|
2023-04-28 22:27:10 +00:00
|
|
|
const message =
|
|
|
|
leaked.size === 0
|
|
|
|
? ""
|
|
|
|
: `The Pulumi runtime detected that ${leaked.size} ${promisePlural} still active\n` +
|
|
|
|
"at the time that the process exited. There are a few ways that this can occur:\n" +
|
|
|
|
" * Not using `await` or `.then` on a Promise returned from a Pulumi API\n" +
|
|
|
|
" * Introducing a cyclic dependency between two Pulumi Resources\n" +
|
|
|
|
" * A bug in the Pulumi Runtime\n" +
|
|
|
|
"\n" +
|
|
|
|
"Leaving promises active is probably not what you want. If you are unsure about\n" +
|
|
|
|
"why you are seeing this message, re-run your program " +
|
|
|
|
"with the `PULUMI_DEBUG_PROMISE_LEAKS`\n" +
|
|
|
|
"environment variable. The Pulumi runtime will then print out additional\n" +
|
|
|
|
"debug information about the leaked promises.";
|
2020-09-24 02:06:26 +00:00
|
|
|
|
|
|
|
if (debugPromiseLeaks) {
|
|
|
|
for (const leak of leaked) {
|
|
|
|
console.error("Promise leak detected:");
|
|
|
|
console.error(promiseDebugString(leak));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-01 18:39:09 +00:00
|
|
|
localStore.leakCandidates = new Set();
|
2020-09-24 02:06:26 +00:00
|
|
|
return [leaked, message];
|
|
|
|
}
|
2017-09-10 01:10:13 +00:00
|
|
|
|
2022-09-20 18:54:06 +00:00
|
|
|
/** @internal */
|
2020-09-24 02:06:26 +00:00
|
|
|
export function promiseDebugString(p: Promise<any>): string {
|
2023-04-28 22:27:10 +00:00
|
|
|
return `CONTEXT(${(<any>p)._debugId}): ${(<any>p)._debugCtx}\n` + `STACK_TRACE:\n` + `${(<any>p)._debugStackTrace}`;
|
2017-09-10 01:10:13 +00:00
|
|
|
}
|
Add Promises leak and hang detection
We have an issue in the runtime right now where we serialize closures
asynchronously, meaning we make it possible to form cycles between
resource graphs (something that ought to be impossible in our model,
where resources are "immutable" after creation and cannot form cycles).
Let me tell you a tale of debugging this ...
Well, no, let's not do that. But thankfully I've left behind some
little utilities that might make debugging such a thing easier down
the road. Namely:
* By default, most of our core runtime promises leverage a leak handler
that will log an error message should the process exit with certain
critical unresolved promises. This error message will include some
handy context (like whether it was an input promise) as well as a
stack trace for its point of creation.
* Optionally, with a flag in runtime/debuggable.ts, you may wire up
a hang detector, for situations where we may want to detect this
situation sooner than process exit, using the regular message loop.
This uses a defined timeout, prints the same diagnostics as the
leak detector when a hang is detected, and is disabled by default.
2017-09-07 01:35:20 +00:00
|
|
|
|
2020-09-24 02:06:26 +00:00
|
|
|
let promiseId = 0;
|
|
|
|
|
2017-09-22 01:15:29 +00:00
|
|
|
/**
|
|
|
|
* debuggablePromise optionally wraps a promise with some goo to make it easier to debug common problems.
|
2020-01-26 17:06:35 +00:00
|
|
|
* @internal
|
2017-09-22 01:15:29 +00:00
|
|
|
*/
|
2018-11-25 02:57:17 +00:00
|
|
|
export function debuggablePromise<T>(p: Promise<T>, ctx: any): Promise<T> {
|
2022-09-01 18:39:09 +00:00
|
|
|
const localStore = state.getStore();
|
2018-05-03 06:29:20 +00:00
|
|
|
// Whack some stack onto the promise. Leave them non-enumerable to avoid awkward rendering.
|
2020-09-24 02:06:26 +00:00
|
|
|
Object.defineProperty(p, "_debugId", { writable: true, value: promiseId });
|
2018-05-03 06:29:20 +00:00
|
|
|
Object.defineProperty(p, "_debugCtx", { writable: true, value: ctx });
|
|
|
|
Object.defineProperty(p, "_debugStackTrace", { writable: true, value: new Error().stack });
|
Eliminate Computed/Property in favor of Promises
As part of pulumi/pulumi-fabric#331, we've been exploring just using
undefined to indicate that a property value is absent during planning.
We also considered blocking the message loop to simplify the overall
programming model, so that all asynchrony is hidden.
It turns out ThereBeDragons :dragon_face: anytime you try to block the
message loop. So, we aren't quite sure about that bit.
But the part we are convicted about is that this Computed/Property
model is far too complex. Furthermore, it's very close to promises, and
yet frustratingly so far away. Indeed, the original thinking in
pulumi/pulumi-fabric#271 was simply to use promises, but we wanted to
encourage dataflow styles, rather than control flow. But we muddied up
our thinking by worrying about awaiting a promise that would never resolve.
It turns out we can achieve a middle ground: resolve planning promises to
undefined, so that they don't lead to hangs, but still use promises so
that asynchrony is explicit in the system. This also avoids blocking the
message loop. Who knows, this may actually be a fine final destination.
2017-09-20 14:40:09 +00:00
|
|
|
|
2020-09-24 02:06:26 +00:00
|
|
|
promiseId++;
|
|
|
|
|
2018-05-17 22:32:39 +00:00
|
|
|
if (!leakDetectorScheduled) {
|
|
|
|
process.on("exit", (code: number) => {
|
|
|
|
// Only print leaks if we're exiting normally. Otherwise, it could be a crash, which of
|
|
|
|
// course yields things that look like "leaks".
|
2018-09-06 17:52:05 +00:00
|
|
|
//
|
|
|
|
// process.exitCode is undefined unless set, in which case it's the exit code that was
|
|
|
|
// passed to process.exit.
|
|
|
|
if ((process.exitCode === undefined || process.exitCode === 0) && !log.hasErrors()) {
|
2020-09-24 02:06:26 +00:00
|
|
|
const [leaks, message] = leakedPromises();
|
|
|
|
if (leaks.size === 0) {
|
2018-05-17 22:32:39 +00:00
|
|
|
// No leaks - proceed with the exit.
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-09-24 02:06:26 +00:00
|
|
|
// If we haven't opted-in to the debug error message, print a more user-friendly message.
|
|
|
|
if (!debugPromiseLeaks) {
|
|
|
|
console.error(message);
|
2017-09-07 22:19:08 +00:00
|
|
|
}
|
Eliminate Computed/Property in favor of Promises
As part of pulumi/pulumi-fabric#331, we've been exploring just using
undefined to indicate that a property value is absent during planning.
We also considered blocking the message loop to simplify the overall
programming model, so that all asynchrony is hidden.
It turns out ThereBeDragons :dragon_face: anytime you try to block the
message loop. So, we aren't quite sure about that bit.
But the part we are convicted about is that this Computed/Property
model is far too complex. Furthermore, it's very close to promises, and
yet frustratingly so far away. Indeed, the original thinking in
pulumi/pulumi-fabric#271 was simply to use promises, but we wanted to
encourage dataflow styles, rather than control flow. But we muddied up
our thinking by worrying about awaiting a promise that would never resolve.
It turns out we can achieve a middle ground: resolve planning promises to
undefined, so that they don't lead to hangs, but still use promises so
that asynchrony is explicit in the system. This also avoids blocking the
message loop. Who knows, this may actually be a fine final destination.
2017-09-20 14:40:09 +00:00
|
|
|
|
2018-05-17 22:32:39 +00:00
|
|
|
// Fail the deployment if we leaked any promises.
|
|
|
|
process.exitCode = 1;
|
|
|
|
}
|
Eliminate Computed/Property in favor of Promises
As part of pulumi/pulumi-fabric#331, we've been exploring just using
undefined to indicate that a property value is absent during planning.
We also considered blocking the message loop to simplify the overall
programming model, so that all asynchrony is hidden.
It turns out ThereBeDragons :dragon_face: anytime you try to block the
message loop. So, we aren't quite sure about that bit.
But the part we are convicted about is that this Computed/Property
model is far too complex. Furthermore, it's very close to promises, and
yet frustratingly so far away. Indeed, the original thinking in
pulumi/pulumi-fabric#271 was simply to use promises, but we wanted to
encourage dataflow styles, rather than control flow. But we muddied up
our thinking by worrying about awaiting a promise that would never resolve.
It turns out we can achieve a middle ground: resolve planning promises to
undefined, so that they don't lead to hangs, but still use promises so
that asynchrony is explicit in the system. This also avoids blocking the
message loop. Who knows, this may actually be a fine final destination.
2017-09-20 14:40:09 +00:00
|
|
|
});
|
2018-05-17 22:32:39 +00:00
|
|
|
leakDetectorScheduled = true;
|
Add Promises leak and hang detection
We have an issue in the runtime right now where we serialize closures
asynchronously, meaning we make it possible to form cycles between
resource graphs (something that ought to be impossible in our model,
where resources are "immutable" after creation and cannot form cycles).
Let me tell you a tale of debugging this ...
Well, no, let's not do that. But thankfully I've left behind some
little utilities that might make debugging such a thing easier down
the road. Namely:
* By default, most of our core runtime promises leverage a leak handler
that will log an error message should the process exit with certain
critical unresolved promises. This error message will include some
handy context (like whether it was an input promise) as well as a
stack trace for its point of creation.
* Optionally, with a flag in runtime/debuggable.ts, you may wire up
a hang detector, for situations where we may want to detect this
situation sooner than process exit, using the regular message loop.
This uses a defined timeout, prints the same diagnostics as the
leak detector when a hang is detected, and is disabled by default.
2017-09-07 01:35:20 +00:00
|
|
|
}
|
|
|
|
|
2018-05-17 22:32:39 +00:00
|
|
|
// Add this promise to the leak candidates list, and schedule it for removal if it resolves.
|
2022-09-01 18:39:09 +00:00
|
|
|
localStore.leakCandidates.add(p);
|
2023-04-28 22:27:10 +00:00
|
|
|
return p
|
|
|
|
.then((val: any) => {
|
|
|
|
localStore.leakCandidates.delete(p);
|
|
|
|
return val;
|
|
|
|
})
|
|
|
|
.catch((err: any) => {
|
|
|
|
localStore.leakCandidates.delete(p);
|
|
|
|
err.promise = p;
|
|
|
|
throw err;
|
|
|
|
});
|
Add Promises leak and hang detection
We have an issue in the runtime right now where we serialize closures
asynchronously, meaning we make it possible to form cycles between
resource graphs (something that ought to be impossible in our model,
where resources are "immutable" after creation and cannot form cycles).
Let me tell you a tale of debugging this ...
Well, no, let's not do that. But thankfully I've left behind some
little utilities that might make debugging such a thing easier down
the road. Namely:
* By default, most of our core runtime promises leverage a leak handler
that will log an error message should the process exit with certain
critical unresolved promises. This error message will include some
handy context (like whether it was an input promise) as well as a
stack trace for its point of creation.
* Optionally, with a flag in runtime/debuggable.ts, you may wire up
a hang detector, for situations where we may want to detect this
situation sooner than process exit, using the regular message loop.
This uses a defined timeout, prints the same diagnostics as the
leak detector when a hang is detected, and is disabled by default.
2017-09-07 01:35:20 +00:00
|
|
|
}
|
|
|
|
|
2017-09-22 01:15:29 +00:00
|
|
|
/**
|
|
|
|
* errorString produces a string from an error, conditionally including additional diagnostics.
|
2020-01-26 17:06:35 +00:00
|
|
|
* @internal
|
2017-09-22 01:15:29 +00:00
|
|
|
*/
|
2017-09-15 23:38:52 +00:00
|
|
|
export function errorString(err: Error): string {
|
2018-04-07 15:02:59 +00:00
|
|
|
if (err.stack) {
|
2017-09-15 23:38:52 +00:00
|
|
|
return err.stack;
|
|
|
|
}
|
|
|
|
return err.toString();
|
|
|
|
}
|