From 9986e10c07f2067bf3cd573219d668eb93d0a4a8 Mon Sep 17 00:00:00 2001 From: Somnath Ghosh Date: Tue, 16 May 2023 15:45:46 -0700 Subject: [PATCH] [Do Not Merge][CPPS][gcs][PoC] Create UVM without networking setup, set network during container creation --- internal/guest/network/network.go | 1 + .../guest/runtime/hcsv2/sandbox_container.go | 28 +++++---- internal/guest/runtime/hcsv2/uvm.go | 37 +++++++++++- .../guest/runtime/hcsv2/workload_container.go | 57 +++++++++++++++++++ 4 files changed, 106 insertions(+), 17 deletions(-) diff --git a/internal/guest/network/network.go b/internal/guest/network/network.go index 68f7c1bef1..3106e96061 100644 --- a/internal/guest/network/network.go +++ b/internal/guest/network/network.go @@ -34,6 +34,7 @@ const maxDNSSearches = 6 // GenerateEtcHostsContent generates a /etc/hosts file based on `hostname`. func GenerateEtcHostsContent(ctx context.Context, hostname string) string { + log.G(ctx).Debug("Inside guest-network-GenerateEtcHostsContent") _, span := oc.StartSpan(ctx, "network::GenerateEtcHostsContent") defer span.End() span.AddAttributes(trace.StringAttribute("hostname", hostname)) diff --git a/internal/guest/runtime/hcsv2/sandbox_container.go b/internal/guest/runtime/hcsv2/sandbox_container.go index 864d7221c5..0223cef0a9 100644 --- a/internal/guest/runtime/hcsv2/sandbox_container.go +++ b/internal/guest/runtime/hcsv2/sandbox_container.go @@ -7,13 +7,11 @@ import ( "context" "os" "path/filepath" - "strings" oci "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "go.opencensus.io/trace" - "github.com/Microsoft/hcsshim/internal/guest/network" specInternal "github.com/Microsoft/hcsshim/internal/guest/spec" "github.com/Microsoft/hcsshim/internal/oc" "github.com/Microsoft/hcsshim/pkg/annotations" @@ -64,20 +62,20 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) ( } // Write the hosts - sandboxHostsContent := network.GenerateEtcHostsContent(ctx, hostname) - sandboxHostsPath := getSandboxHostsPath(id) - if err := os.WriteFile(sandboxHostsPath, []byte(sandboxHostsContent), 0644); err != nil { - return errors.Wrapf(err, "failed to write sandbox hosts to %q", sandboxHostsPath) - } + // sandboxHostsContent := network.GenerateEtcHostsContent(ctx, hostname) + // sandboxHostsPath := getSandboxHostsPath(id) + // if err := os.WriteFile(sandboxHostsPath, []byte(sandboxHostsContent), 0644); err != nil { + // return errors.Wrapf(err, "failed to write sandbox hosts to %q", sandboxHostsPath) + // } // Write resolv.conf - ns, err := getNetworkNamespace(getNetworkNamespaceID(spec)) - if err != nil { - return err - } - var searches, servers []string - for _, n := range ns.Adapters() { - if len(n.DNSSuffix) > 0 { + // ns, err := getNetworkNamespace(getNetworkNamespaceID(spec)) + // if err != nil { + // return err + // } + // var searches, servers []string + // for _, n := range ns.Adapters() { + /* if len(n.DNSSuffix) > 0 { searches = network.MergeValues(searches, strings.Split(n.DNSSuffix, ",")) } if len(n.DNSServerList) > 0 { @@ -91,7 +89,7 @@ func setupSandboxContainerSpec(ctx context.Context, id string, spec *oci.Spec) ( sandboxResolvPath := getSandboxResolvPath(id) if err := os.WriteFile(sandboxResolvPath, []byte(resolvContent), 0644); err != nil { return errors.Wrap(err, "failed to write sandbox resolv.conf") - } + }*/ // User.Username is generally only used on Windows, but as there's no (easy/fast at least) way to grab // a uid:gid pairing for a username string on the host, we need to defer this work until we're here in the diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 8770534dff..12e29ee432 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -357,12 +357,14 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM return nil, err } case "container": + namespaceID = getNetworkNamespaceID(settings.OCISpecification) sid, ok := settings.OCISpecification.Annotations[annotations.KubernetesSandboxID] sandboxID = sid if !ok || sid == "" { return nil, errors.Errorf("unsupported 'io.kubernetes.cri.sandbox-id': '%s'", sid) } if err := setupWorkloadContainerSpec(ctx, sid, id, settings.OCISpecification); err != nil { + log.G(ctx).Debug("UVM:CreateContainer: Failed in setupWorkloadContainerSpec") return nil, err } @@ -381,8 +383,10 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM } }() if err := policy.ExtendPolicyWithNetworkingMounts(sandboxID, h.securityPolicyEnforcer, settings.OCISpecification); err != nil { + log.G(ctx).Debug("UVM:CreateContainer: Failed in ExtendPolicyWithNetworkingMounts") return nil, err } + log.G(ctx).Debug("UVM:CreateContainer: Completed set up workload container spec") default: return nil, errors.Errorf("unsupported 'io.kubernetes.cri.container-type': '%s'", criType) } @@ -505,7 +509,9 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM return nil, errors.Wrapf(err, "failed to flush writer for config.json at: '%s'", configFile) } + log.G(ctx).Debug("UVM:CreateContainer: Calling h.rtime.CreateContainer") con, err := h.rtime.CreateContainer(id, settings.OCIBundlePath, nil) + log.G(ctx).Debug("UVM:CreateContainer: Completed h.rtime.CreateContainer") if err != nil { return nil, errors.Wrapf(err, "failed to create container") } @@ -518,7 +524,13 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM c.initProcess = newProcess(c, settings.OCISpecification.Process, init, uint32(c.container.Pid()), true) // Sandbox or standalone, move the networks to the container namespace - if criType == "sandbox" || !isCRI { + // However in case of preprovisioned VMs, we create the network in case + // of workload containers, so we need to move the networks to the container + // namespace. + log.G(ctx).Debug("Logging namespace ID:", namespaceID) + log.G(ctx).Debug("criType =", criType) + if criType != "sandbox" || !isCRI { + log.G(ctx).Debug("Moving networks to namespace") ns, err := getNetworkNamespace(namespaceID) if isCRI && err != nil { return nil, err @@ -532,6 +544,8 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM return nil, err } } + } else { + log.G(ctx).Debug("Skip moving networks to namespace") } c.setStatus(containerCreated) @@ -540,6 +554,8 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM func (h *Host) modifyHostSettings(ctx context.Context, containerID string, req *guestrequest.ModificationRequest) (err error) { switch req.ResourceType { + case guestresource.ResourceTypeSCSIDevice: + return modifySCSIDevice(ctx, req.RequestType, req.Settings.(*guestresource.SCSIDevice)) case guestresource.ResourceTypeMappedVirtualDisk: mvd := req.Settings.(*guestresource.LCOWMappedVirtualDisk) // find the actual controller number on the bus and update the incoming request. @@ -939,6 +955,23 @@ func newInvalidRequestTypeError(rt guestrequest.RequestType) error { return errors.Errorf("the RequestType %q is not supported", rt) } +func modifySCSIDevice( + ctx context.Context, + rt guestrequest.RequestType, + msd *guestresource.SCSIDevice, +) error { + switch rt { + case guestrequest.RequestTypeRemove: + cNum, err := scsi.ActualControllerNumber(ctx, msd.Controller) + if err != nil { + return err + } + return scsi.UnplugDevice(ctx, cNum, msd.Lun) + default: + return newInvalidRequestTypeError(rt) + } +} + func modifyMappedVirtualDisk( ctx context.Context, rt guestrequest.RequestType, @@ -979,7 +1012,7 @@ func modifyMappedVirtualDisk( return err } } - return scsi.UnplugDevice(ctx, mvd.Controller, mvd.Lun) + return nil default: return newInvalidRequestTypeError(rt) } diff --git a/internal/guest/runtime/hcsv2/workload_container.go b/internal/guest/runtime/hcsv2/workload_container.go index b3a470029d..b1053c441b 100644 --- a/internal/guest/runtime/hcsv2/workload_container.go +++ b/internal/guest/runtime/hcsv2/workload_container.go @@ -17,6 +17,8 @@ import ( specInternal "github.com/Microsoft/hcsshim/internal/guest/spec" "github.com/Microsoft/hcsshim/internal/guestpath" "github.com/Microsoft/hcsshim/internal/hooks" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/guest/network" "github.com/Microsoft/hcsshim/internal/oc" "github.com/Microsoft/hcsshim/pkg/annotations" ) @@ -94,7 +96,48 @@ func specHasGPUDevice(spec *oci.Spec) bool { return false } +func setupNetworking(ctx context.Context, hostname, id string, spec *oci.Spec) (err error) { + // Write the hosts + sandboxHostsContent := network.GenerateEtcHostsContent(ctx, hostname) + sandboxHostsPath := getSandboxHostsPath(id) + log.G(ctx).WithField("sandboxHostsPath", sandboxHostsPath).Debug("Printing sandboxHosts Path") + log.G(ctx).Debug("SandboxHostsContent ", sandboxHostsContent) + if err := os.WriteFile(sandboxHostsPath, []byte(sandboxHostsContent), 0644); err != nil { + return errors.Wrapf(err, "failed to write sandbox hosts to %q", sandboxHostsPath) + } + + // Write resolv.conf + ns, err := getNetworkNamespace(getNetworkNamespaceID(spec)) + log.G(ctx).Debug("Network namespace", ns) + if err != nil { + log.G(ctx).Debug("Error getting network namespace") + return err + } + var searches, servers []string + for _, n := range ns.Adapters() { + if len(n.DNSSuffix) > 0 { + searches = network.MergeValues(searches, strings.Split(n.DNSSuffix, ",")) + } + if len(n.DNSServerList) > 0 { + servers = network.MergeValues(servers, strings.Split(n.DNSServerList, ",")) + } + } + resolvContent, err := network.GenerateResolvConfContent(ctx, searches, servers, nil) + if err != nil { + return errors.Wrap(err, "failed to generate sandbox resolv.conf content") + } + sandboxResolvPath := getSandboxResolvPath(id) + log.G(ctx).Debug("sandboxResolvPath", sandboxResolvPath) + log.G(ctx).Debug("sandboxResolvContent", resolvContent) + if err := os.WriteFile(sandboxResolvPath, []byte(resolvContent), 0644); err != nil { + return errors.Wrap(err, "failed to write sandbox resolv.conf") + } + + return nil +} + func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci.Spec) (err error) { + log.G(ctx).Debug("Inside setupWorkloadContainerSpec") ctx, span := oc.StartSpan(ctx, "hcsv2::setupWorkloadContainerSpec") defer span.End() defer func() { oc.SetSpanStatus(span, err) }() @@ -107,6 +150,18 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci. return errors.Errorf("workload container must not change hostname: %s", spec.Hostname) } + hostname, err := os.Hostname() + if hostname == "" { + return errors.Wrap(err, "failed to get hostname") + } + log.G(ctx).WithField("hostname", hostname).Debug("Printing hostname"); + + // Set up networking + log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> Setting up networking"); + err = setupNetworking(ctx, hostname, sbid, spec); + log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> Completed setting up networking"); + + log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> UpdateSandboxMounts") // update any sandbox mounts with the sandboxMounts directory path and create files if err = updateSandboxMounts(sbid, spec); err != nil { return errors.Wrapf(err, "failed to update sandbox mounts for container %v in sandbox %v", id, sbid) @@ -116,6 +171,7 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci. return errors.Wrapf(err, "failed to update hugepages mounts for container %v in sandbox %v", id, sbid) } + log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> GenerateWorkloadContainerNetworkMounts") // Add default mounts for container networking (e.g. /etc/hostname, /etc/hosts), // if spec didn't override them explicitly. networkingMounts := specInternal.GenerateWorkloadContainerNetworkMounts(sbid, spec) @@ -171,5 +227,6 @@ func setupWorkloadContainerSpec(ctx context.Context, sbid, id string, spec *oci. // Clear the windows section as we dont want to forward to runc spec.Windows = nil + log.G(ctx).Debug("Inside setupWorkloadContainerSpec -> Completed") return nil }