pulumi/sdk/go/common/resource/plugin/rpc.go

673 lines
24 KiB
Go

// Copyright 2016-2021, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package plugin
import (
"fmt"
"reflect"
"sort"
"google.golang.org/protobuf/types/known/structpb"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/archive"
"github.com/pulumi/pulumi/sdk/v3/go/common/resource/asset"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
"github.com/pulumi/pulumi/sdk/v3/go/common/util/logging"
)
// MarshalOptions controls the marshaling of RPC structures.
type MarshalOptions struct {
Label string // an optional label for debugging.
SkipNulls bool // true to skip nulls altogether in the resulting map.
KeepUnknowns bool // true if we are keeping unknown values (otherwise we skip them).
RejectUnknowns bool // true if we should return errors on unknown values. Takes precedence over KeepUnknowns.
ElideAssetContents bool // true if we are eliding the contents of assets.
ComputeAssetHashes bool // true if we are computing missing asset hashes on the fly.
KeepSecrets bool // true if we are keeping secrets (otherwise we replace them with their underlying value).
RejectAssets bool // true if we should return errors on Asset and Archive values.
KeepResources bool // true if we are keeping resoures (otherwise we return raw urn).
SkipInternalKeys bool // true to skip internal property keys (keys that start with "__") in the resulting map.
KeepOutputValues bool // true if we are keeping output values.
UpgradeToOutputValues bool // true if secrets and unknowns should be upgraded to output values.
WorkingDirectory string // the optional working directory to use when serializing assets & archives.
}
const (
// UnknownBoolValue is a sentinel indicating that a bool property's value is not known, because it depends on
// a computation with values whose values themselves are not yet known (e.g., dependent upon an output property).
UnknownBoolValue = "1c4a061d-8072-4f0a-a4cb-0ff528b18fe7"
// UnknownNumberValue is a sentinel indicating that a number property's value is not known, because it depends on
// a computation with values whose values themselves are not yet known (e.g., dependent upon an output property).
UnknownNumberValue = "3eeb2bf0-c639-47a8-9e75-3b44932eb421"
// UnknownStringValue is a sentinel indicating that a string property's value is not known, because it depends on
// a computation with values whose values themselves are not yet known (e.g., dependent upon an output property).
UnknownStringValue = "04da6b54-80e4-46f7-96ec-b56ff0331ba9"
// UnknownArrayValue is a sentinel indicating that an array property's value is not known, because it depends on
// a computation with values whose values themselves are not yet known (e.g., dependent upon an output property).
UnknownArrayValue = "6a19a0b0-7e62-4c92-b797-7f8e31da9cc2"
// UnknownAssetValue is a sentinel indicating that an asset property's value is not known, because it depends on
// a computation with values whose values themselves are not yet known (e.g., dependent upon an output property).
UnknownAssetValue = "030794c1-ac77-496b-92df-f27374a8bd58"
// UnknownArchiveValue is a sentinel indicating that an archive property's value is not known, because it depends
// on a computation with values whose values themselves are not yet known (e.g., dependent upon an output property).
UnknownArchiveValue = "e48ece36-62e2-4504-bad9-02848725956a"
// UnknownObjectValue is a sentinel indicating that an archive property's value is not known, because it depends
// on a computation with values whose values themselves are not yet known (e.g., dependent upon an output property).
UnknownObjectValue = "dd056dcd-154b-4c76-9bd3-c8f88648b5ff"
)
// MarshalProperties marshals a resource's property map as a "JSON-like" protobuf structure.
func MarshalProperties(props resource.PropertyMap, opts MarshalOptions) (*structpb.Struct, error) {
fields := make(map[string]*structpb.Value)
for _, key := range props.StableKeys() {
v := props[key]
logging.V(9).Infof("Marshaling property for RPC[%s]: %s=%v", opts.Label, key, v)
if opts.SkipNulls && v.IsNull() {
logging.V(9).Infof("Skipping null property for RPC[%s]: %s (as requested)", opts.Label, key)
} else if opts.SkipInternalKeys && resource.IsInternalPropertyKey(key) {
logging.V(9).Infof("Skipping internal property for RPC[%s]: %s (as requested)", opts.Label, key)
} else {
m, err := MarshalPropertyValue(key, v, opts)
if err != nil {
return nil, err
} else if m != nil {
fields[string(key)] = m
}
}
}
return &structpb.Struct{
Fields: fields,
}, nil
}
// MarshalPropertyValue marshals a single resource property value into its "JSON-like" value representation.
func MarshalPropertyValue(key resource.PropertyKey, v resource.PropertyValue,
opts MarshalOptions,
) (*structpb.Value, error) {
if v.IsNull() {
return MarshalNull(opts), nil
} else if v.IsBool() {
return &structpb.Value{
Kind: &structpb.Value_BoolValue{
BoolValue: v.BoolValue(),
},
}, nil
} else if v.IsNumber() {
return &structpb.Value{
Kind: &structpb.Value_NumberValue{
NumberValue: v.NumberValue(),
},
}, nil
} else if v.IsString() {
return MarshalString(v.StringValue(), opts), nil
} else if v.IsArray() {
var elems []*structpb.Value
for _, elem := range v.ArrayValue() {
e, err := MarshalPropertyValue(key, elem, opts)
if err != nil {
return nil, err
}
if e != nil {
elems = append(elems, e)
}
}
return &structpb.Value{
Kind: &structpb.Value_ListValue{
ListValue: &structpb.ListValue{Values: elems},
},
}, nil
} else if v.IsAsset() {
if opts.RejectAssets {
return nil, fmt.Errorf("unexpected Asset property value for %q", key)
}
return MarshalAsset(v.AssetValue(), opts)
} else if v.IsArchive() {
if opts.RejectAssets {
return nil, fmt.Errorf("unexpected Asset Archive property value for %q", key)
}
return MarshalArchive(v.ArchiveValue(), opts)
} else if v.IsObject() {
obj, err := MarshalProperties(v.ObjectValue(), opts)
if err != nil {
return nil, err
}
return MarshalStruct(obj, opts), nil
} else if v.IsComputed() {
if opts.RejectUnknowns {
return nil, fmt.Errorf("unexpected unknown property value for %q", key)
} else if opts.KeepUnknowns {
if opts.KeepOutputValues && opts.UpgradeToOutputValues {
output := resource.NewObjectProperty(resource.PropertyMap{
resource.SigKey: resource.NewStringProperty(resource.OutputValueSig),
})
return MarshalPropertyValue(key, output, opts)
}
return marshalUnknownProperty(v.Input().Element, opts), nil
}
return nil, nil // return nil and the caller will ignore it.
} else if v.IsOutput() {
if !opts.KeepOutputValues {
result := v.OutputValue().Element
if !v.OutputValue().Known {
// Unknown outputs are marshaled the same as Computed.
result = resource.MakeComputed(resource.NewStringProperty(""))
}
if v.OutputValue().Secret {
result = resource.MakeSecret(result)
}
return MarshalPropertyValue(key, result, opts)
}
obj := resource.PropertyMap{
resource.SigKey: resource.NewStringProperty(resource.OutputValueSig),
}
if v.OutputValue().Known {
obj["value"] = v.OutputValue().Element
}
if v.OutputValue().Secret {
obj["secret"] = resource.NewBoolProperty(v.OutputValue().Secret)
}
if len(v.OutputValue().Dependencies) > 0 {
deps := make([]resource.PropertyValue, len(v.OutputValue().Dependencies))
for i, dep := range v.OutputValue().Dependencies {
deps[i] = resource.NewStringProperty(string(dep))
}
obj["dependencies"] = resource.NewArrayProperty(deps)
}
output := resource.NewObjectProperty(obj)
return MarshalPropertyValue(key, output, opts)
} else if v.IsSecret() {
if !opts.KeepSecrets {
logging.V(5).Infof("marshalling secret value as raw value as opts.KeepSecrets is false")
return MarshalPropertyValue(key, v.SecretValue().Element, opts)
}
if opts.KeepOutputValues && opts.UpgradeToOutputValues {
output := resource.NewObjectProperty(resource.PropertyMap{
resource.SigKey: resource.NewStringProperty(resource.OutputValueSig),
"secret": resource.NewBoolProperty(true),
"value": v.SecretValue().Element,
})
return MarshalPropertyValue(key, output, opts)
}
secret := resource.NewObjectProperty(resource.PropertyMap{
resource.SigKey: resource.NewStringProperty(resource.SecretSig),
"value": v.SecretValue().Element,
})
return MarshalPropertyValue(key, secret, opts)
} else if v.IsResourceReference() {
ref := v.ResourceReferenceValue()
if !opts.KeepResources {
val := string(ref.URN)
if !ref.ID.IsNull() {
return MarshalPropertyValue(key, ref.ID, opts)
}
logging.V(5).Infof("marshalling resource value as raw URN or ID as opts.KeepResources is false")
return MarshalString(val, opts), nil
}
m := resource.PropertyMap{
resource.SigKey: resource.NewStringProperty(resource.ResourceReferenceSig),
"urn": resource.NewStringProperty(string(ref.URN)),
}
if id, hasID := ref.IDString(); hasID {
m["id"] = resource.NewStringProperty(id)
}
if ref.PackageVersion != "" {
m["packageVersion"] = resource.NewStringProperty(ref.PackageVersion)
}
return MarshalPropertyValue(key, resource.NewObjectProperty(m), opts)
}
contract.Failf("Unrecognized property value in RPC[%s] for %q: %v (type=%v)",
opts.Label, key, v.V, reflect.TypeOf(v.V))
return nil, nil
}
// marshalUnknownProperty marshals an unknown property in a way that lets us recover its type on the other end.
func marshalUnknownProperty(elem resource.PropertyValue, opts MarshalOptions) *structpb.Value {
// Normal cases, these get sentinels.
if elem.IsBool() {
return MarshalString(UnknownBoolValue, opts)
} else if elem.IsNumber() {
return MarshalString(UnknownNumberValue, opts)
} else if elem.IsString() {
return MarshalString(UnknownStringValue, opts)
} else if elem.IsArray() {
return MarshalString(UnknownArrayValue, opts)
} else if elem.IsAsset() {
return MarshalString(UnknownAssetValue, opts)
} else if elem.IsArchive() {
return MarshalString(UnknownArchiveValue, opts)
} else if elem.IsObject() {
return MarshalString(UnknownObjectValue, opts)
}
// If for some reason we end up with a recursive computed/output, just keep digging.
if elem.IsComputed() {
return marshalUnknownProperty(elem.Input().Element, opts)
} else if elem.IsOutput() {
return marshalUnknownProperty(elem.OutputValue().Element, opts)
}
// Finally, if a null, we can guess its value! (the one and only...)
if elem.IsNull() {
return MarshalNull(opts)
}
contract.Failf("Unexpected output/computed property element in RPC[%s]: %v", opts.Label, elem)
return nil
}
// UnmarshalProperties unmarshals a "JSON-like" protobuf structure into a new resource property map.
func UnmarshalProperties(props *structpb.Struct, opts MarshalOptions) (resource.PropertyMap, error) {
result := make(resource.PropertyMap)
// First sort the keys so we enumerate them in order (in case errors happen, we want determinism).
var keys []string
if props != nil {
for k := range props.Fields {
keys = append(keys, k)
}
sort.Strings(keys)
}
// And now unmarshal every field it into the map.
for _, key := range keys {
pk := resource.PropertyKey(key)
v, err := UnmarshalPropertyValue(pk, props.Fields[key], opts)
if err != nil {
return nil, err
} else if v != nil {
logging.V(9).Infof("Unmarshaling property for RPC[%s]: %s=%v", opts.Label, key, v)
if opts.SkipNulls && v.IsNull() {
logging.V(9).Infof("Skipping unmarshaling for RPC[%s]: %s is null", opts.Label, key)
} else if opts.SkipInternalKeys && resource.IsInternalPropertyKey(pk) {
logging.V(9).Infof("Skipping unmarshaling for RPC[%s]: %s is internal", opts.Label, key)
} else {
result[pk] = *v
}
}
}
return result, nil
}
// UnmarshalPropertyValue unmarshals a single "JSON-like" value into a new property value.
func UnmarshalPropertyValue(key resource.PropertyKey, v *structpb.Value,
opts MarshalOptions,
) (*resource.PropertyValue, error) {
contract.Assertf(v != nil, "a value is required")
switch v.Kind.(type) {
case *structpb.Value_NullValue:
m := resource.NewNullProperty()
return &m, nil
case *structpb.Value_BoolValue:
m := resource.NewBoolProperty(v.GetBoolValue())
return &m, nil
case *structpb.Value_NumberValue:
m := resource.NewNumberProperty(v.GetNumberValue())
return &m, nil
case *structpb.Value_StringValue:
// If it's a string, it could be an unknown property, or just a regular string.
s := v.GetStringValue()
if unk, isunk := unmarshalUnknownPropertyValue(s, opts); isunk {
if opts.RejectUnknowns {
return nil, fmt.Errorf("unexpected unknown property value for %q", key)
} else if opts.KeepUnknowns {
return &unk, nil
}
return nil, nil
}
m := resource.NewStringProperty(s)
return &m, nil
case *structpb.Value_ListValue:
lst := v.GetListValue()
elems := make([]resource.PropertyValue, len(lst.GetValues()))
for i, elem := range lst.GetValues() {
e, err := UnmarshalPropertyValue(key, elem, opts)
if err != nil {
return nil, err
} else if e != nil {
if i == len(elems) {
elems = append(elems, *e)
} else {
elems[i] = *e
}
}
}
m := resource.NewArrayProperty(elems)
return &m, nil
case *structpb.Value_StructValue:
// Start by unmarshaling.
obj, err := UnmarshalProperties(v.GetStructValue(), opts)
if err != nil {
return nil, err
}
// Before returning it as an object, check to see if it's a known recoverable type.
objmap := obj.Mappable()
sig, hasSig := objmap[string(resource.SigKey)]
if !hasSig {
// This is a weakly-typed object map.
m := resource.NewObjectProperty(obj)
return &m, nil
}
switch sig {
case asset.AssetSig:
if opts.RejectAssets {
return nil, fmt.Errorf("unexpected Asset property value for %q", key)
}
asset, isasset, err := asset.Deserialize(objmap)
if err != nil {
return nil, err
}
// This can only be false with a non-nil error if there is a signature match. We've already verified the
// signature.
contract.Assertf(isasset, "value must be an asset")
if opts.ComputeAssetHashes {
if opts.WorkingDirectory == "" {
if err = asset.EnsureHash(); err != nil {
return nil, fmt.Errorf("failed to compute asset hash for %q: %w", key, err)
}
} else {
if err = asset.EnsureHashWithWD(opts.WorkingDirectory); err != nil {
return nil, fmt.Errorf("failed to compute asset hash for %q: %w", key, err)
}
}
}
m := resource.NewAssetProperty(asset)
return &m, nil
case archive.ArchiveSig:
if opts.RejectAssets {
return nil, fmt.Errorf("unexpected Asset Archive property value for %q", key)
}
archive, isarchive, err := archive.Deserialize(objmap)
if err != nil {
return nil, err
}
// This can only be false with a non-nil error if there is a signature match. We've already verified the
// signature.
contract.Assertf(isarchive, "value must be an archive")
if opts.ComputeAssetHashes {
if opts.WorkingDirectory == "" {
if err = archive.EnsureHash(); err != nil {
return nil, fmt.Errorf("failed to compute archive hash for %q: %w", key, err)
}
} else {
if err = archive.EnsureHashWithWD(opts.WorkingDirectory); err != nil {
return nil, fmt.Errorf("failed to compute archive hash for %q: %w", key, err)
}
}
}
m := resource.NewArchiveProperty(archive)
return &m, nil
case resource.SecretSig:
value, ok := obj["value"]
if !ok {
return nil, fmt.Errorf("malformed RPC secret: missing value for %q", key)
}
return unmarshalSecretPropertyValue(value, opts), nil
case resource.ResourceReferenceSig:
urn, ok := obj["urn"]
if !ok {
return nil, fmt.Errorf("malformed resource reference for %q: missing urn", key)
}
if !urn.IsString() {
return nil, fmt.Errorf("malformed resource reference for %q: urn not a string", key)
}
id, hasID := "", false
if idProp, ok := obj["id"]; ok {
hasID = true
switch {
case idProp.IsString():
id = idProp.StringValue()
case idProp.IsComputed():
// Leave the ID empty to indicate that it is unknown.
case idProp.IsOutput():
if idProp.OutputValue().Known {
if !idProp.OutputValue().Element.IsString() {
return nil, fmt.Errorf("malformed resource reference for %q: id not a string", key)
}
id = idProp.OutputValue().Element.StringValue()
}
default:
return nil, fmt.Errorf("malformed resource reference for %q: id not a string", key)
}
}
var packageVersion string
if packageVersionProp, ok := obj["packageVersion"]; ok {
if !packageVersionProp.IsString() {
return nil, fmt.Errorf("malformed resource reference for %q: packageVersion not a string", key)
}
packageVersion = packageVersionProp.StringValue()
}
if !opts.KeepResources {
value := urn.StringValue()
if hasID {
isIDUnknown := id == ""
if isIDUnknown && opts.KeepUnknowns {
v := structpb.Value{
Kind: &structpb.Value_StringValue{StringValue: UnknownStringValue},
}
return UnmarshalPropertyValue(key, &v, opts)
}
value = id
}
r := resource.NewStringProperty(value)
return &r, nil
}
var ref resource.PropertyValue
if hasID {
ref = resource.MakeCustomResourceReference(resource.URN(urn.StringValue()), resource.ID(id), packageVersion)
} else {
ref = resource.MakeComponentResourceReference(resource.URN(urn.StringValue()), packageVersion)
}
return &ref, nil
case resource.OutputValueSig:
value, known := obj["value"]
var secret bool
if secretProp, ok := obj["secret"]; ok {
if !secretProp.IsBool() {
return nil, fmt.Errorf("malformed output value for %q: secret not a bool", key)
}
secret = secretProp.BoolValue()
}
if !opts.KeepOutputValues {
result := &value
if !known {
result, err = UnmarshalPropertyValue(key, &structpb.Value{
Kind: &structpb.Value_StringValue{StringValue: UnknownStringValue},
}, opts)
if err != nil {
return nil, err
}
}
if secret && result != nil {
result = unmarshalSecretPropertyValue(*result, opts)
}
return result, nil
}
var dependencies []resource.URN
if dependenciesProp, ok := obj["dependencies"]; ok {
if !dependenciesProp.IsArray() {
return nil, fmt.Errorf("malformed output value for %q: dependencies not an array", key)
}
dependencies = make([]resource.URN, len(dependenciesProp.ArrayValue()))
for i, dep := range dependenciesProp.ArrayValue() {
if !dep.IsString() {
return nil, fmt.Errorf(
"malformed output value for %q: element in dependencies not a string", key)
}
dependencies[i] = resource.URN(dep.StringValue())
}
}
output := resource.NewOutputProperty(resource.Output{
Element: value,
Known: known,
Secret: secret,
Dependencies: dependencies,
})
return &output, nil
default:
return nil, fmt.Errorf("unrecognized signature '%v' in property map for %q", sig, key)
}
default:
contract.Failf("Unrecognized structpb value kind in RPC[%s] for %q: %v", opts.Label, key, reflect.TypeOf(v.Kind))
return nil, nil
}
}
func unmarshalUnknownPropertyValue(s string, opts MarshalOptions) (resource.PropertyValue, bool) {
var elem resource.PropertyValue
var unknown bool
switch s {
case UnknownBoolValue:
elem, unknown = resource.NewBoolProperty(false), true
case UnknownNumberValue:
elem, unknown = resource.NewNumberProperty(0), true
case UnknownStringValue:
elem, unknown = resource.NewStringProperty(""), true
case UnknownArrayValue:
elem, unknown = resource.NewArrayProperty([]resource.PropertyValue{}), true
case UnknownAssetValue:
elem, unknown = resource.NewAssetProperty(&asset.Asset{}), true
case UnknownArchiveValue:
elem, unknown = resource.NewArchiveProperty(&archive.Archive{}), true
case UnknownObjectValue:
elem, unknown = resource.NewObjectProperty(make(resource.PropertyMap)), true
}
if unknown {
if opts.KeepOutputValues && opts.UpgradeToOutputValues {
return resource.NewOutputProperty(resource.Output{
Element: elem,
}), true
}
comp := resource.Computed{Element: elem}
return resource.NewComputedProperty(comp), true
}
return resource.PropertyValue{}, false
}
func unmarshalSecretPropertyValue(v resource.PropertyValue, opts MarshalOptions) *resource.PropertyValue {
if !opts.KeepSecrets {
logging.V(5).Infof("unmarshalling secret as raw value, as opts.KeepSecrets is false")
return &v
}
var s resource.PropertyValue
if opts.KeepOutputValues && opts.UpgradeToOutputValues {
s = resource.NewOutputProperty(resource.Output{
Element: v,
Secret: true,
Known: true,
})
} else {
s = resource.MakeSecret(v)
}
return &s
}
// MarshalNull marshals a nil to its protobuf form.
func MarshalNull(opts MarshalOptions) *structpb.Value {
return &structpb.Value{
Kind: &structpb.Value_NullValue{
NullValue: structpb.NullValue_NULL_VALUE,
},
}
}
// MarshalString marshals a string to its protobuf form.
func MarshalString(s string, opts MarshalOptions) *structpb.Value {
return &structpb.Value{
Kind: &structpb.Value_StringValue{
StringValue: s,
},
}
}
// MarshalStruct marshals a struct for use in a protobuf field where a value is expected.
func MarshalStruct(obj *structpb.Struct, opts MarshalOptions) *structpb.Value {
return &structpb.Value{
Kind: &structpb.Value_StructValue{
StructValue: obj,
},
}
}
// MarshalAsset marshals an asset into its wire form for resource provider plugins.
func MarshalAsset(v *asset.Asset, opts MarshalOptions) (*structpb.Value, error) {
// If we are not providing access to an asset's contents, we simply need to record the fact that this asset existed.
// Serialize the asset with only its hash (if present).
if opts.ElideAssetContents {
v = &asset.Asset{Hash: v.Hash}
} else {
// Ensure a hash is present if needed.
if v.Hash == "" && opts.ComputeAssetHashes {
if opts.WorkingDirectory == "" {
if err := v.EnsureHash(); err != nil {
return nil, fmt.Errorf("failed to compute asset hash: %w", err)
}
} else {
if err := v.EnsureHashWithWD(opts.WorkingDirectory); err != nil {
return nil, fmt.Errorf("failed to compute asset hash: %w", err)
}
}
}
}
// To marshal an asset, we need to first serialize it, and then marshal that.
sera := v.Serialize()
serap := resource.NewPropertyMapFromMap(sera)
pk := resource.PropertyKey(v.URI)
return MarshalPropertyValue(pk, resource.NewObjectProperty(serap), opts)
}
// MarshalArchive marshals an archive into its wire form for resource provider plugins.
func MarshalArchive(v *archive.Archive, opts MarshalOptions) (*structpb.Value, error) {
// If we are not providing access to an asset's contents, we simply need to record the fact that this asset existed.
// Serialize the asset with only its hash (if present).
if opts.ElideAssetContents {
v = &archive.Archive{Hash: v.Hash}
} else {
// Ensure a hash is present if needed.
if v.Hash == "" && opts.ComputeAssetHashes {
if opts.WorkingDirectory == "" {
if err := v.EnsureHash(); err != nil {
return nil, fmt.Errorf("failed to compute archive hash: %w", err)
}
} else {
if err := v.EnsureHashWithWD(opts.WorkingDirectory); err != nil {
return nil, fmt.Errorf("failed to compute archive hash: %w", err)
}
}
}
}
// To marshal an archive, we need to first serialize it, and then marshal that.
sera := v.Serialize()
serap := resource.NewPropertyMapFromMap(sera)
pk := resource.PropertyKey(v.URI)
return MarshalPropertyValue(pk, resource.NewObjectProperty(serap), opts)
}