test: added tests

added tests and more documentation in docstrings. Added a TODO.md file to track the new things to do

Author: d4rkstar

.github/cisetup.sh

@@ -18,3 +18,4 @@
 #
 sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
 sudo apt-get -y install curl wget jq
+pip install uv

.github/workflows/check.yml

@@ -34,3 +34,12 @@ jobs:
           submodules: recursive
       - name: License   
         uses: apache/skywalking-eyes@main
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+      - name: Setup
+        run: bash .github/cisetup.sh
+      - name: Unit Tests
+        run: task utest
+        continue-on-error: false

.licenserc.yaml

@@ -28,6 +28,7 @@ header:
     - 'LICENSE'
     - 'NOTICE'
     - 'DISCLAIMER'
+    - 'deploy/samples/requirements.txt'
     - '**/*.json'
     - '**/*.service'
     - '**/*.txt'

TODO.md

@@ -0,0 +1,28 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  ~
+-->
+# TODO
+
+## Tests
+Add integration and unit tests
+
+## Various
+
+- [ ] `openserverless.common.whis_user_data.py` - Add `with_` blocks for other new OpenServerless Services
+- [ ] `openserverless.common.whisk_user_generator` - Check if `generate_whisk_user_yaml` is complete

Taskfile.yml

@@ -126,3 +126,12 @@ tasks:
         BASEIMG=$(task base-image-name)
         IMG="$BASEIMG:{{.TAG}}"
         kind load docker-image $IMG --name=nuvolaris
+
+  utest:
+    cmds:
+      - |
+        for test in openserverless/common/{{.T}}*.py
+        do  echo "*** [{{.KUBE}}] $test"
+            uv run python3 -m doctest -o ELLIPSIS $test {{.CLI_ARGS}}
+        done
+    silent: true

deploy/buildkit/buildkitd.toml

@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
 # =========================
 # Worker OCI (rootlesskit)
 # =========================

openserverless/common/kube_api_client.py

@@ -22,25 +22,15 @@
 import logging
 
 from base64 import b64decode, b64encode
-from .validation import is_empty_arg
 
+from openserverless.common.utils import join_host_port
 from openserverless.config.app_config import AppConfig
 from openserverless.error.config_exception import ConfigException
 
 SERVICE_HOST_ENV_NAME = "KUBERNETES_SERVICE_HOST"
 SERVICE_PORT_ENV_NAME = "KUBERNETES_SERVICE_PORT"
 SERVICE_TOKEN_FILENAME = "/var/run/secrets/kubernetes.io/serviceaccount/token"
 SERVICE_CERT_FILENAME = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-
-
-def _join_host_port(host, port):
-    template = "%s:%s"
-    host_requires_bracketing = ":" in host or "%" in host
-    if host_requires_bracketing:
-        template = "[%s]:%s"
-    return template % (host, port)
-
-
 class KubeApiClient:
 
     def __init__(self, environ=os.environ):
@@ -50,11 +40,6 @@ def __init__(self, environ=os.environ):
         self._load_incluster_config()
 
     def _parse_b64(self, encoded_str):
-        """
-        Decode b64 encoded string
-        param: encoded_str a Base64 encoded string
-        return: decoded string
-        """
         try:
             return b64decode(encoded_str).decode()
         except:
@@ -73,7 +58,7 @@ def _load_incluster_config(self):
         ):
             raise ConfigException("Service host/port is not set.")
 
-        self.host = "https://" + _join_host_port(
+        self.host = "https://" + join_host_port(
             self._environ.get(SERVICE_HOST_ENV_NAME),
             self._environ.get(SERVICE_PORT_ENV_NAME),
         )
@@ -254,7 +239,13 @@ def get_config_map(self, cm_name, namespace="nuvolaris"):
             return None
 
     def post_config_map(self, cm_name, file_or_dir, namespace="nuvolaris"):
-        
+        """        
+        Create a ConfigMap from a file or directory.
+        :param cm_name: Name of the ConfigMap.
+        :param file_or_dir: Path to the file or directory containing the data.
+        :param namespace: Namespace where the ConfigMap will be created.
+        :return: The created ConfigMap or None if failed.
+        """
         if not os.path.exists(file_or_dir):
             raise ConfigException(f"File or directory {file_or_dir} does not exist.")
 
@@ -301,6 +292,12 @@ def post_config_map(self, cm_name, file_or_dir, namespace="nuvolaris"):
             return None
 
     def delete_config_map(self, cm_name, namespace="nuvolaris"):
+        """
+        Delete a ConfigMap by name.
+        :param cm_name: Name of the ConfigMap to delete.
+        :param namespace: Namespace where the ConfigMap is located.
+        :return: True if deletion was successful, False otherwise.
+        """
         url = f"{self.host}/api/v1/namespaces/{namespace}/configmaps/{cm_name}"
         headers = {"Authorization": self.token}
 
@@ -416,8 +413,14 @@ def delete_secret(self, secret_name, namespace="nuvolaris"):
             logging.error(f"delete_secret {ex}")
             return False
 
-    # --- CREA JOB ---
-    def post_job(self, job_name, job_manifest, namespace="nuvolaris"):        
+    def post_job(self, job_name, job_manifest, namespace="nuvolaris"):
+        """
+        Create a Kubernetes job.
+        :param job_name: Name of the job.
+        :param job_manifest: Dictionary containing the job manifest. 
+        :param namespace: Namespace where the job will be created.
+        :return: The created job or None if failed.
+        """
         url = f"{self.host}/apis/batch/v1/namespaces/{namespace}/jobs"
         headers = {"Authorization": self.token}
         try:
@@ -437,8 +440,13 @@ def post_job(self, job_name, job_manifest, namespace="nuvolaris"):
             logging.error(f"post_job {ex}")
             return None
 
-    # --- OTTIENI POD ---
     def get_pod_by_job_name(self, job_name, namespace="nuvolaris"):
+        """
+        Get the pod name associated with a job by its name.
+        :param job_name: Name of the job.
+        :param namespace: Namespace where the job is located.
+        :return: The pod name if found, None otherwise.
+        """
         url = f"{self.host}/api/v1/namespaces/{namespace}/pods"
         headers = {"Authorization": self.token}
         try:
@@ -466,17 +474,26 @@ def get_pod_by_job_name(self, job_name, namespace="nuvolaris"):
             logging.error(f"get_pod_by_job_name {ex}")
             return None
 
-    # --- LEGGI LOG POD ---
     def stream_pod_logs(self, pod_name, namespace="nuvolaris"):
+        """
+        Stream logs from a specific pod.
+        :param pod_name: Name of the pod to stream logs from.
+        :param namespace: Namespace where the pod is located.
+        """
         url = f"{self.host}/api/v1/namespaces/{namespace}/pods/{pod_name}/log?follow=true"
         headers = {"Authorization": self.token}
         with req.get(url, headers=headers, verify=self.ssl_ca_cert, stream=True) as r:
             for line in r.iter_lines():
                 if line:
                     print(line.decode())
 
-    # --- CHECK STATUS JOB ---
     def check_job_status(self, job_name, namespace="nuvolaris"):
+        """
+        Check the status of a job by its name.
+        :param job_name: Name of the job to check.
+        :param namespace: Namespace where the job is located.
+        :return: True if the job has succeeded, False otherwise.
+        """
         url = f"{self.host}/apis/batch/v1/namespaces/{namespace}/jobs/{job_name}"
         headers = {"Authorization": self.token}
         try:

openserverless/common/openwhisk_authorize.py

@@ -33,7 +33,6 @@ class OpenwhiskAuthorize:
     def __init__(self, environ=os.environ):
         self._db = CouchDB()
         self._environ = environ
-        
 
     def encode(self, username, password):
         """Returns an HTTP basic authentication encrypted string given a valid
@@ -46,23 +45,52 @@ def encode(self, username, password):
         return f"Basic {b64encode(username_password.encode()).decode()}"
 
     def _parse_b64(self, encoded_str):
+        """
+        Parse a base64 encoded string and return the username and password.
+        If the string is not base64 encoded, it will try to split it by ':'.
+        Raises DecodeError if the string cannot be decoded or parsed.
+        >>> oa = OpenwhiskAuthorize()
+        >>> oa._parse_b64("dXNlcm5hbWU6cGFzc3dvcmQ=")
+        ('username', 'password')
+        >>> oa._parse_b64("username:password")
+        ('username', 'password')
+        >>> oa._parse_b64("invalid_base64_string")
+        Traceback (most recent call last):
+            ...
+        openserverless.error.api_error.DecodeError: authentication token does not seems to be b64 encoded
+        """
+        username = None
+        password = None
         try:
-            username, password = b64decode(encoded_str).decode().split(":", 1)
+            decoded = b64decode(encoded_str)
+
+            credentials = decoded.decode()
+            if credentials.count(":") != 1:
+                raise DecodeError("authentication token does not seems to be b64 encoded")
+            username, password = credentials.split(":", 1)
         except:
             # fallback in case the token is not bas64 encoded
-            username, password = encoded_str.split(":", 1)
+            if encoded_str.count(":") == 1:
+                username, password = encoded_str.split(":", 1)
 
-            if not username or not password:
-                raise DecodeError(
-                    "authentication token does not seems to be b64 encoded"
-                )
+        if not username or not password:
+            raise DecodeError("authentication token does not seems to be b64 encoded")
 
         return username, password
 
     def decode(self, encoded_str):
         """Decode an encrypted HTTP basic authentication string. Returns a tuple of
         the form (username, password), and raises a DecodeError exception if
         nothing could be decoded.
+        >>> oa = OpenwhiskAuthorize()
+        >>> oa.decode("Basic dXNlcm5hbWU6cGFzc3dvcmQ=")
+        ('username', 'password')
+        >>> oa.decode("dXNlcm5hbWU6cGFzc3dvcmQ=")
+        ('username', 'password')
+        >>> oa.decode("invalid_base64_string")
+        Traceback (most recent call last):
+            ...
+        openserverless.error.api_error.DecodeError: authentication token does not seems to be b64 encoded
         """
         split = encoded_str.strip().split(" ")
 

openserverless/common/utils.py

@@ -21,6 +21,19 @@ def env_to_dict(user_data, key="env"):
 
     Keyword arguments:
     key -- the key to extract the env from
+
+    >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}, {"key": "VAR2", "value": "value2"}]})
+    {'VAR1': 'value1', 'VAR2': 'value2'}
+    >>> env_to_dict({"env": []})
+    {}
+    >>> env_to_dict({"other_key": [{"key": "VAR1", "value": "value1"}]}, key="other_key")
+    {'VAR1': 'value1'}
+    >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}]}, key="env")
+    {'VAR1': 'value1'}
+    >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}, {"key": "VAR2", "value": "value2"}]}, key="non_existent_key")
+    {}
+    >>> env_to_dict({"env": [{"key": "VAR1", "value": "value1"}]}, key="env")
+    {'VAR1': 'value1'}
     """
     body = {}
     if key in user_data:
@@ -37,9 +50,41 @@ def env_to_dict(user_data, key="env"):
 def dict_to_env(env):
     """
     converts an env to a key/pair suitable for user_data storage
+
+    >>> dict_to_env({"VAR1": "value1", "VAR2": "value2"})
+    [{'key': 'VAR1', 'value': 'value1'}, {'key': 'VAR2', 'value': 'value2'}]
+    >>> dict_to_env({})
+    []
     """
     body = []
     for key in env:
         body.append({"key": key, "value": env[key]})
 
-    return body
+    return body
+
+def join_host_port(host, port):
+    """
+    Join host and port into a URL format.
+    >>> join_host_port("localhost", "8080")
+    'localhost:8080'
+    >>> join_host_port("localhost", 8080)
+    'localhost:8080'
+    >>> join_host_port("localhost", "80")
+    'localhost:80'
+    >>> join_host_port("localhost", "abcd")
+    Traceback (most recent call last):
+        ...
+    ValueError: Port must be numeric
+
+    """
+    template = "%s:%s"
+    try:
+        port_int = int(port)
+        port = str(port_int)
+    except (ValueError, TypeError):
+        raise ValueError("Port must be numeric")
+    
+    host_requires_bracketing = ":" in host or "%" in host
+    if host_requires_bracketing:
+        template = "[%s]:%s"
+    return template % (host, port)

openserverless/common/validation.py

@@ -21,6 +21,20 @@
 def is_valid_username(username):
     """
     Verifies the given username follows nuvolaris rule
+    >>> is_valid_username("bruno")
+    True
+    >>> is_valid_username("bruno123")
+    True
+    >>> is_valid_username("bruno-123")
+    False
+    >>> is_valid_username("bruno_123")
+    False
+    >>> is_valid_username("bruno@123")
+    False
+    >>> is_valid_username("bruno 123")
+    False
+    >>> is_valid_username("brun")
+    False
     """
     pat = re.compile(r"^[a-z0-9]{5,60}(?:[a-z0-9])?$")
     if re.fullmatch(pat, username):
@@ -35,6 +49,10 @@ def is_empty_arg(args, arg_name):
     param: args
     param: arg_name
     return: True if the argument is not contained in the input args array or if it is an empty string value
+    >>> is_empty_arg({"arg1": "value1"}, "arg1")
+    False
+    >>> is_empty_arg({"arg1": "value1"}, "arg2")
+    True
     """
 
     if arg_name not in args: