Implement obtain python packages from EnvironmentConfigs and from Compositions

Add support for Capabilities and Required Schemas

Author: iciclespider

crossplane/pythonic/composite.py

@@ -104,11 +104,13 @@ def __init__(self, crossplane_v1, request, logger):
         )
         self.response = protobuf.Message(None, 'response', response.DESCRIPTOR, response)
         self.logger = logger
+        self.capabilities = Capabilities(self.request.meta.capabilities)
         self.parameters = self.request.input.parameters
         self.credentials = Credentials(self.request)
         self.context = self.response.context
         self.environment = self.context['apiextensions.crossplane.io/environment']
         self.requireds = Requireds(self)
+        self.schemas = Schemas(self)
         self.resources = Resources(self)
         self.autoReady = True
         self.usages = False
@@ -123,6 +125,7 @@ def __init__(self, crossplane_v1, request, logger):
         self.metadata = self.observed.metadata
         self.spec = self.observed.spec
         self.status = self.desired.status
+        self.output = self.response.output
         self.conditions = Conditions(observed, self.response)
         self.results = Results(self.response)
         self.events = Results(self.response) # Deprecated, use self.results
@@ -136,6 +139,30 @@ async def compose(self):
         raise NotImplementedError()
 
 
+class Capabilities:
+    def __init__(self, capabilities):
+        self._capabilities = capabilities
+
+    def __bool__(self):
+        return fnv1.CAPABILITY_CAPABILITIES in self._capabilities
+
+    @property
+    def requireds(self):
+        return fnv1.CAPABILITY_REQUIRED_RESOURCES in self._capabilities if self else None
+
+    @property
+    def schemas(self):
+        return fnv1.CAPABILITY_REQUIRED_SCHEMAS in self._capabilities if self else None
+
+    @property
+    def credentials(self):
+        return fnv1.CAPABILITY_CREDENTIALS in self._capabilities if self else None
+
+    @property
+    def conditions(self):
+        return fnv1.CAPABILITY_CONDITIONS in self._capabilities if self else None
+
+
 class Credentials:
     def __init__(self, request):
         self.__dict__['_request'] = request
@@ -558,6 +585,150 @@ def __bool__(self):
         return bool(self.observed)
 
 
+class Schemas:
+    def __init__(self, composite):
+        self._composite = composite
+        self._cache = {}
+
+    def __getattr__(self, key):
+        return self[key]
+
+    def __getitem__(self, key):
+        schema = self._cache.get(key)
+        if not schema:
+            schema = Schema(self._composite, key)
+            self._cache[key] = schema
+        return schema
+
+    def __bool__(self):
+        return bool(len(self))
+
+    def __len__(self):
+        names = set()
+        for name, schema in self._composite.request.required_schemas:
+            names.add(name)
+        for name, selector in self._composite.response.requirements.schemas:
+            names.add(name)
+        return len(names)
+
+    def __contains__(self, key):
+        if key in self._composite.request.required_schemas:
+            return True
+        if key in self._composite.response.requirements.schemas:
+            return True
+        return False
+
+    def __iter__(self):
+        names = set()
+        for name, schema in self._composite.request.required_schemas:
+            names.add(name)
+        for name, selector in self._composite.response.requirements.schemas:
+            names.add(name)
+        for name in sorted(names):
+            yield name, self[name]
+
+
+class Schema:
+    def __init__(self, composite, name):
+        self.name = name
+        self._selector = composite.response.requirements.schemas[name]
+        self._schema = composite.request.required_schemas[name].openapi_v3
+
+    def __call__(self, kind=_notset, apiVersion=_notset):
+        self._selector()
+        if kind != _notset:
+            # Allow for apiVersion in the first arg and kind in the second arg
+            if '/' in kind or kind == 'v1':
+                if apiVersion != _notset:
+                    self.kind = apiVersion
+                apiVersion = kind
+            else:
+                self.kind = kind
+        if apiVersion != _notset:
+            self.apiVersion = apiVersion
+        return self
+
+    @property
+    def apiVersion(self):
+        return self._selector.api_version
+
+    @apiVersion.setter
+    def apiVersion(self, apiVersion):
+        self._selector.api_version = apiVersion
+
+    @property
+    def kind(self):
+        return self._selector.kind
+
+    @kind.setter
+    def kind(self, kind):
+        self._selector.kind = kind
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        pass
+
+    def __aenter__(self):
+        return self
+
+    def __aexit__(self, exc_type, exc_value, traceback):
+        pass
+
+    def __getattr__(self, key):
+        return self[key]
+
+    def __getitem__(self, key):
+        return self._schema[key]
+
+    def __bool__(self):
+        return bool(self._schema)
+
+    def __len__(self):
+        return len(self._schema)
+
+    def __contains__(self, item):
+        return item in self._schema
+
+    def __iter__(self):
+        for key, value in self._schema:
+            yield key, value
+
+    def __hash__(self):
+        return hash(self._schema)
+
+    def __eq__(self, other):
+        if instance(other, Schema):
+            other = other._schema
+        return self._schema == other
+
+    def __str__(self):
+        return str(self._schema)
+
+    def __format__(self, spec='yaml'):
+        return format(self,_schema, spec)
+
+
+class RequiredResource:
+    def __init__(self, name, ix, resource):
+        self.name = name
+        self.ix = ix
+        self.observed = resource.resource
+        self.apiVersion = self.observed.apiVersion
+        self.kind = self.observed.kind
+        self.metadata = self.observed.metadata
+        self.spec = self.observed.spec
+        self.type = self.observed.type
+        self.data = self.observed.data
+        self.status = self.observed.status
+        self.conditions = Conditions(resource)
+        self.connection = self.observed.connection_details
+
+    def __bool__(self):
+        return bool(self.observed)
+
+
 class Conditions:
     def __init__(self, observed, response=None):
         self._observed = observed

crossplane/pythonic/function.py

@@ -121,8 +121,13 @@ async def run_function(self, request):
         except Exception as e:
             return self.fatal(request, logger, 'Compose', e)
 
-        if requireds := self.get_requireds(step, composite):
-            logger.debug(f"Requireds requested: {','.join(requireds)}")
+        schemas = self.get_schemas(step, composite)
+        requireds = self.get_requireds(step, composite)
+        if schemas or requireds:
+            if schemas:
+                logger.debug(f"Required schemas: {','.join(schemas)}")
+            if requireds:
+                logger.debug(f"Required resources: {','.join(requireds)}")
         else:
             self.process_usages(composite)
             self.process_unknowns(composite)
@@ -155,11 +160,21 @@ def fatal(self, request, logger, message, exception=None):
             ]
         )
 
+    def get_schemas(self, step, composite):
+        schemas = []
+        for name, schema in composite.schemas:
+            if len(schema.kind) and len(schema.apiVersion):
+                s = pythonic.Map(kind=schema.kind, apiVersion=schema.apiVersion)
+                if s != step.schemas[name]:
+                    step.schemas[name] = s
+                    schemas.append(name)
+        return schemas
+
     def get_requireds(self, step, composite):
         requireds = []
         for name, required in composite.requireds:
-            if len(required.apiVersion) and len(required.kind):
-                r = pythonic.Map(apiVersion=required.apiVersion, kind=required.kind)
+            if len(required.kind) and len(required.apiVersion):
+                r = pythonic.Map(kind=required.kind, apiVersion=required.apiVersion)
                 if len(required.namespace):
                     r.namespace = required.namespace
                 if len(required.matchName):

crossplane/pythonic/grpc.py

@@ -45,6 +45,12 @@ def add_parser_arguments(cls, parser):
         parser.add_argument(
             '--packages',
             action='store_true',
+            dest='packages_configmaps',
+            help='Discover python packages from function-pythonic ConfigMaps, deprecated use --packages-configmaps'
+        )
+        parser.add_argument(
+            '--packages-configmaps',
+            action='store_true',
             help='Discover python packages from function-pythonic ConfigMaps.'
         )
         parser.add_argument(
@@ -57,7 +63,17 @@ def add_parser_arguments(cls, parser):
             action='append',
             default=[],
             metavar='NAMESPACE',
-            help='Namespaces to discover function-pythonic ConfigMaps in, default is cluster wide.',
+            help='Namespaces to discover function-pythonic ConfigMaps and Secrets in, default is cluster wide.',
+        )
+        parser.add_argument(
+            '--packages-environmentconfigs',
+            action='store_true',
+            help='Also Discover python packages from function-pythonic EnvironmentConfigs.'
+        )
+        parser.add_argument(
+            '--packages-compositions',
+            action='store_true',
+            help='Also Discover python packages from function-pythonic Compositions.'
         )
         parser.add_argument(
             '--packages-dir',
@@ -75,7 +91,9 @@ def initialize(self):
         if not self.args.tls_certs_dir and not self.args.insecure:
             print('Either --tls-certs-dir or --insecure must be specified', file=sys.stderr)
             sys.exit(1)
-
+        if (self.args.packages_environmentconfigs or self.args.packages_compositions) and self.args.packages_namespace:
+            print('--packages-namespace cannot be used with --packages-environment-configs or --packages-compositions', file=sys.stderr)
+            sys.exit(1)
         if self.args.pip_install:
             import pip._internal.cli.main
             pip._internal.cli.main.main(['install', '--user', *shlex.split(self.args.pip_install)])
@@ -108,15 +126,18 @@ async def run(self):
             )
         await grpc_server.start()
 
-        if self.args.packages:
+        if self.args.packages_configmaps or self.args.packages_secrets or self.args.packages_environmentconfigs or self.args.packages_compositions:
             from . import packages
             async with asyncio.TaskGroup() as tasks:
                 tasks.create_task(grpc_server.wait_for_termination())
                 tasks.create_task(packages.operator(
                     grpc_server,
                     grpc_runner,
+                    self.args.packages_configmaps,
                     self.args.packages_secrets,
                     self.args.packages_namespace,
+                    self.args.packages_environmentconfigs,
+                    self.args.packages_compositions,
                     self.args.packages_dir,
                 ))
         else: