Merge pull request #1361 from JosepSampe/lithops-dev

[Standalone] Fix runtime metadata extraction

docs/source/compute_config/aws_batch.md

@@ -71,13 +71,13 @@ In summary, you can use one of the following settings:
         region : <REGION_NAME>
         execution_role: <EXECUTION_ROLE_ARN>
         instance_role: <INSTANCE_ROLE_ARN>
+        job_role: <JOB_ROLE_ARN>
         subnets:
             - <SUBNET_ID_1>
             - <SUBNET_ID_2>
             - ...
         security_groups:
             - <SECURITY_GROUP_1>
-            - <SECURITY_GROUP_2>
             - ...
     ```
 
@@ -94,13 +94,13 @@ In summary, you can use one of the following settings:
     aws_batch:
         execution_role: <EXECUTION_ROLE_ARN>
         instance_role: <INSTANCE_ROLE_ARN>
+        job_role: <JOB_ROLE_ARN>
         subnets:
             - <SUBNET_ID_1>
             - <SUBNET_ID_2>
             - ...
         security_groups:
             - <SECURITY_GROUP_1>
-            - <SECURITY_GROUP_2>
             - ...
     ```
 
@@ -126,7 +126,7 @@ In summary, you can use one of the following settings:
 | aws_batch  | security_groups  |  | yes | List of Security groups to attach for ECS task containers. By default, you can use a security group that accepts all outbound traffic but blocks all inbound traffic. |
 | aws_batch  | subnets          |  | yes | List of subnets from a VPC where to deploy the ECS task containers. Note that if you are using a **private subnet**, you can set `assign_public_ip` to `false` but make sure containers can reach other AWS services like ECR, Secrets service, etc., by, for example, using a NAT gateway. If you are using a **public subnet** you must set `assign_public_up` to `true` |
 | aws_batch  | region      |  | no | Region name (like `us-east-1`) where to deploy the ECS cluster. Lithops will use the region set under the `aws` section if it is not set here |
-| aws_batch  | assign_public_ip | `true` | no | Assing public IPs to ECS task containers. Set to `true` if the tasks are being deployed in a public subnet. Set to `false` when deploying on a private subnet. |
+| aws_batch  | assign_public_ip | `true` | no | Assign public IPs to ECS task containers. Set to `true` if the tasks are being deployed in a public subnet. Set to `false` when deploying on a private subnet. |
 | aws_batch  | runtime          | `default_runtime-v3X` | no | Runtime name |
 | aws_batch  | runtime_timeout  | 180 | no | Runtime timeout |
 | aws_batch  | runtime_memory   | 1024 | no | Runtime memory |

docs/source/compute_config/aws_lambda.md

@@ -43,7 +43,7 @@ python3 -m pip install lithops[aws]
 
 5. Go back to **IAM** and navigate to **Roles** tab. Click **Create role**.
 
-6. Choose **Lambda** on the use case list and click **Next: Permissions**. Select the policy created before (`lithops-policy`). Click **Next: Tags** and **Next: Review**. Type a role name, for example `lithops-lambda-execution-role`. Click on *Create Role*.
+6. Choose **Lambda** on the use case list and click **Next: Permissions**. Select the policy created before (`lithops-policy`). Click **Next: Tags** and **Next: Review**. Type a role name, for example `lambdaLithopsExecutionRole`. Click on *Create Role*.
 
 ## AWS Credential setup
 
@@ -93,7 +93,7 @@ In summary, you can use one of the following settings:
 
 | Group | Key | Default | Mandatory | Additional info |
 | --- | --- | --- | --- | --- |
-| aws_lambda | execution_role |  | yes | ARN of the execution role created at step 3. You can find it in the Role page at the *Roles* list in the *IAM* section (e.g. `arn:aws:iam::1234567890:role/lithops-execution-role` |
+| aws_lambda | execution_role |  | yes | ARN of the execution role created at step 3. You can find it in the Role page at the *Roles* list in the *IAM* section (e.g. `arn:aws:iam::1234567890:role/lambdaLithopsExecutionRole` |
 | aws_lambda | region |  | no | Region where Lambda functions will be invoked (e.g. `us-east-1`). Lithops will use the `region` set under the `aws` section if it is not set here |
 | aws_lambda | max_workers | 1000 | no | Max number of workers per `FunctionExecutor()` |
 | aws_lambda | worker_processes | 1 | no | Number of Lithops processes within a given worker. This can be used to parallelize function activations within a worker |

lithops/config.py

@@ -186,10 +186,6 @@ def default_config(config_file=None, config_data=None, config_overwrite={}, load
         sb_config.load_config(config_data)
         config_data['lithops']['chunksize'] = 0
 
-    for key in c.LITHOPS_DEFAULT_CONFIG_KEYS:
-        if key not in config_data['lithops']:
-            config_data['lithops'][key] = c.LITHOPS_DEFAULT_CONFIG_KEYS[key]
-
     if 'chunksize' not in config_data['lithops']:
         config_data['lithops']['chunksize'] = config_data[backend]['worker_processes']
 
@@ -199,6 +195,10 @@ def default_config(config_file=None, config_data=None, config_overwrite={}, load
            and backend != c.LOCALHOST:
             raise Exception(f'Localhost storage backend cannot be used with {backend}')
 
+    for key in c.LITHOPS_DEFAULT_CONFIG_KEYS:
+        if key not in config_data['lithops']:
+            config_data['lithops'][key] = c.LITHOPS_DEFAULT_CONFIG_KEYS[key]
+
     return config_data
 
 

lithops/localhost/config.py

@@ -15,6 +15,7 @@
 import os
 import re
 import sys
+from enum import Enum
 
 
 DEFAULT_CONFIG_KEYS = {
@@ -25,6 +26,23 @@
 LOCALHOST_EXECUTION_TIMEOUT = 3600
 
 
+class LocvalhostEnvironment(Enum):
+    DEFAULT = "default"
+    CONTAINER = "container"
+
+
+def get_environment(runtime_name):
+
+    windows_path_pattern = re.compile(r'^[A-Za-z]:\\.*$')
+    if runtime_name.startswith(('python', '/')) \
+       or windows_path_pattern.match(runtime_name) is not None:
+        environment = LocvalhostEnvironment.DEFAULT
+    else:
+        environment = LocvalhostEnvironment.CONTAINER
+
+    return environment
+
+
 def load_config(config_data):
 
     if 'localhost' not in config_data or not config_data['localhost']:
@@ -41,12 +59,3 @@ def load_config(config_data):
 
     if 'storage' not in config_data['lithops']:
         config_data['lithops']['storage'] = 'localhost'
-
-    windows_path_pattern = re.compile(r'^[A-Za-z]:\\.*$')
-    runtime_name = config_data['localhost']['runtime']
-
-    if runtime_name.startswith(('python', '/')) \
-       or windows_path_pattern.match(runtime_name) is not None:
-        config_data['localhost']['environment'] = 'default'
-    else:
-        config_data['localhost']['environment'] = 'container'

lithops/localhost/v1/localhost.py

@@ -42,6 +42,10 @@
     is_podman,
     is_unix_system
 )
+from lithops.localhost.config import (
+    LocvalhostEnvironment,
+    get_environment
+)
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +63,7 @@ def __init__(self, config):
         logger.debug('Creating Localhost compute client')
         self.config = config
         self.runtime_name = self.config['runtime']
-        self.environment = self.config['environment']
+        self.environment = get_environment(self.runtime_name)
 
         self.env = None
         self.job_queue = queue.Queue()
@@ -79,7 +83,7 @@ def init(self):
         """
         Init tasks for localhost
         """
-        if self.environment == 'default':
+        if self.environment == LocvalhostEnvironment.DEFAULT:
             self.env = DefaultEnvironment(self.config)
         else:
             self.env = ContainerEnvironment(self.config)

lithops/localhost/v2/localhost.py

@@ -46,6 +46,10 @@
     is_podman,
     is_unix_system
 )
+from lithops.localhost.config import (
+    LocvalhostEnvironment,
+    get_environment
+)
 
 logger = logging.getLogger(__name__)
 
@@ -64,7 +68,7 @@ def __init__(self, localhost_config):
         logger.debug('Creating Localhost compute client')
         self.config = localhost_config
         self.runtime_name = self.config['runtime']
-        self.environment = self.config['environment']
+        self.environment = get_environment(self.runtime_name)
 
         self.env = None
         self.job_manager = None
@@ -83,7 +87,7 @@ def init(self):
         """
         Init tasks for localhost
         """
-        if self.environment == 'default':
+        if self.environment == LocvalhostEnvironment.DEFAULT:
             self.env = DefaultEnvironment(self.config)
         else:
             self.env = ContainerEnvironment(self.config)