diff --git a/Pipfile b/Pipfile index 3fe16df..8c6791d 100644 --- a/Pipfile +++ b/Pipfile @@ -9,12 +9,12 @@ jstyleson = "==0.0.2" requests = "==2.31.0" textx = "==4.0.1" js2py = "==0.74" -requests-pkcs12 = "==1.22" +requests-pkcs12 = "==1.24" parsys-requests-unixsocket = "==0.3.1" requests-aws4auth = "==1.2.3" requests-ntlm = "==1.2.0" restrictedpython = "==7.0" -faker = "==22.7.0" +faker = "==23.1.0" requests-hawk = "==1.2.1" pyyaml = "==6.0.1" toml = "==0.10.2" @@ -22,7 +22,7 @@ msal = "==1.26.0" [dev-packages] python-magic = "*" -waitress = "==2.1.1" +waitress = "==3.0.0" flask = "==3.0.2" pyperf = "==2.6.2" pytest-benchmark = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 765225a..6936982 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4246d4edf4268563a8d6419f2da45c6a78d9a16da3039bdfb7354d1f1c5473b9" + "sha256": "5372743ffe7fbca81ab5f1a76971a90e8151abfdd78e0ba0ca3f09bfb491918c" }, "pipfile-spec": 6, "requires": { @@ -233,11 +233,11 @@ }, "faker": { "hashes": [ - "sha256:d12edbac08a82a75ecd588f299f44f12e33f000c15fe414abc417f0836cb51ae", - "sha256:f797529ebeb9bd9e1851106b99e156c9bebe67d2730c8393a1705ed1c864f1bf" + "sha256:60e89e5c0b584e285a7db05eceba35011a241954afdab2853cb246c8a56700a2", + "sha256:b7f76bb1b2ac4cdc54442d955e36e477c387000f31ce46887fb9722a041be60b" ], "index": "pypi", - "version": "==22.7.0" + "version": "==23.1.0" }, "idna": { "hashes": [ @@ -440,11 +440,11 @@ }, "requests-pkcs12": { "hashes": [ - "sha256:7ccfa5109bed1eb3d0710b2fcf0497960170f310d7d3acefbff7af035a459258", - "sha256:e33f6479c84fbe664917feeb37c9949691f3cee1783ffa0580c1f494541707f5" + "sha256:0c3c25c2e5b6794c6a51b4eaed050fd8f7f400ec0807d6dafaaf6c8def018f6e", + "sha256:b312c24d4a460e0e4e02df2f9beb7684f01d270b006db80e6aa957af9cc262b9" ], "index": "pypi", - "version": "==1.22" + "version": "==1.24" }, "restrictedpython": { "hashes": [ @@ -794,11 +794,11 @@ }, "waitress": { "hashes": [ - "sha256:c549f5b2b4afd44d9d97d7cec79f3ef581e25d832827f415dc175327af674aa8", - "sha256:e2e60576cf14a1539da79f7b7ee1e79a71e64f366a0b47db54a15e971f57bb16" + "sha256:005da479b04134cdd9dd602d1ee7c49d79de0537610d653674cc6cbde222b8a1", + "sha256:2a06f242f4ba0cc563444ca3d1998959447477363a2d7e9b8b4d75d35cfd1669" ], "index": "pypi", - "version": "==2.1.1" + "version": "==3.0.0" }, "werkzeug": { "hashes": [ diff --git a/dotextensions/server/handlers/basic_handlers.py b/dotextensions/server/handlers/basic_handlers.py index 2d9184f..889e3dc 100644 --- a/dotextensions/server/handlers/basic_handlers.py +++ b/dotextensions/server/handlers/basic_handlers.py @@ -3,7 +3,8 @@ from requests import RequestException -from dothttp import DotHttpException, Config, HttpDef, MultidefHttp, BaseModelProcessor, UndefinedHttpToExtend, js3py +from dothttp.exceptions import DotHttpException +from dothttp import Config, HttpDef, MultidefHttp, BaseModelProcessor, UndefinedHttpToExtend, js3py from dothttp.request_base import CurlCompiler, RequestCompiler, HttpFileFormatter, dothttp_model from dothttp.__version__ import __version__ from dothttp.parse_models import ScriptType @@ -108,9 +109,6 @@ def get_config(self, command): def get_request_result(self, command, comp: RequestCompiler): comp.load_def() - execution_cls = js3py.ScriptExecutionJs if comp.httpdef.test_script_lang == ScriptType.JAVA_SCRIPT else js3py.ScriptExecutionPython - script_execution = execution_cls(comp.httpdef, comp.property_util) - script_execution.pre_request_script() resp = comp.get_response() if output := comp.httpdef.output: # body = f"Output stored in {output}" @@ -119,7 +117,7 @@ def get_request_result(self, command, comp: RequestCompiler): except Exception as e: output = f"Not!. unhandled error happened : {e}" logger.warning("unable to write because", exc_info=True) - script_result = script_execution.execute_test_script(resp).as_json() + script_result = self.script_execution.execute_test_script(resp).as_json() body = resp.text response_data = { "response": { diff --git a/dotextensions/server/handlers/http2postman.py b/dotextensions/server/handlers/http2postman.py index 1032146..44b680c 100644 --- a/dotextensions/server/handlers/http2postman.py +++ b/dotextensions/server/handlers/http2postman.py @@ -10,7 +10,6 @@ from dotextensions.server.postman2_1 import FormParameterType, File, Mode, AuthType, Variable from dothttp import json_or_array_to_json, UndefinedHttpToExtend, ParameterException, HttpDef, \ request_logger, Payload, APPLICATION_JSON, CONTENT_TYPE, AWS4Auth, dothttp_model -from dothttp.parse_models import NtlmAuthWrap from dothttp.request_base import RequestCompiler from dothttp.utils import json_to_urlencoded_array from . import logger diff --git a/dothttp/__init__.py b/dothttp/__init__.py index 4bfcfad..6c4bb2c 100644 --- a/dothttp/__init__.py +++ b/dothttp/__init__.py @@ -1,79 +1,31 @@ -import logging import mimetypes import os import re import sys +import toml +import yaml + from collections import defaultdict -from dataclasses import dataclass, field -from io import IOBase -from typing import Union, List, Optional, Dict, DefaultDict, Tuple, BinaryIO, Any -from urllib.parse import urlencode, urljoin, uses_relative, uses_netloc, uses_params, uses_query, uses_fragment, \ +from typing import Union, List, Optional, Dict, DefaultDict, Any +from urllib.parse import urljoin, uses_relative, uses_netloc, uses_params, uses_query, uses_fragment, \ urlparse - -from requests import PreparedRequest -from requests.auth import HTTPBasicAuth, HTTPDigestAuth, AuthBase +from functools import lru_cache +from requests.auth import HTTPBasicAuth, HTTPDigestAuth from requests.structures import CaseInsensitiveDict - -try: - from requests_hawk import HawkAuth as RequestsHawkAuth -except BaseException: - RequestsHawkAuth = None - -from .utils import get_real_file_path, triple_or_double_tostring, APPLICATION_JSON, json_to_urlencoded_array - -try: - from requests_aws4auth import AWS4Auth -except ImportError: - AWS4Auth = None -try: - from requests_ntlm import HttpNtlmAuth -except ImportError: - HttpNtlmAuth = None - -try: - import jstyleson as json - from jsonschema import validate -except ImportError: - import json - validate = None -import yaml -import toml from textx import TextXSyntaxError, metamodel_from_file +from .utils import get_real_file_path, triple_or_double_tostring +from .computed import * from .dsl_jsonparser import json_or_array_to_json from .exceptions import * -from .parse_models import AzureAuthCli, AzureAuthType, AzureAuthWrap, MultidefHttp, AuthWrap, DigestAuth, BasicAuth, Line, NtlmAuthWrap, Query, Http, NameWrap, UrlWrap, Header, \ - MultiPartFile, FilesWrap, TripleOrDouble, Payload as ParsePayload, Certificate, P12Certificate, ExtraArg, \ - AWS_REGION_LIST, AWS_SERVICES_LIST, AwsAuthWrap, TestScript, ScriptType, HawkAuth, AzureAuthCertificate, \ +from .parse_models import AzureAuthCli, AzureAuthType, AzureAuthWrap, MultidefHttp, AuthWrap, Http, \ + Certificate, P12Certificate, \ + AWS_REGION_LIST, AWS_SERVICES_LIST, TestScript, ScriptType, AzureAuthCertificate, \ AzureAuthDeviceCode, AzureAuthServicePrincipal from .property_schema import property_schema from .property_util import PropertyProvider -try: - from .azure_auth import AzureAuth -except: - # this is for dothttp-wasm, where msal most likely not installed - AzureAuth = None - -try: - import magic -except ImportError: - magic = None - -base_logger = logging.getLogger("dothttp") -request_logger = logging.getLogger("request") -curl_logger = logging.getLogger("curl") - -MIME_TYPE_JSON = "application/json" -FORM_URLENCODED = "application/x-www-form-urlencoded" -MULTIPART_FORM_INPUT = "multipart/form-data" -TEXT_PLAIN = "text/plain" - -CONTENT_TYPE = 'content-type' - -UNIX_SOCKET_SCHEME = "http+unix" - -BASEIC_AUTHORIZATION_HEADER = "Authorization" - +from .constants import * +from . import js3py def install_unix_socket_scheme(): uses_relative.append(UNIX_SOCKET_SCHEME) @@ -95,283 +47,6 @@ def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -@dataclass -class Config: - curl: bool - property_file: Union[str, None] - properties: List[str] - env: list - debug: bool - file: str - info: bool - no_cookie: bool - format: bool - stdout: bool = False - experimental: bool = False - target: str = field(default_factory=lambda: '1') - content: str = None - - -@dataclass -class Payload: - data: Optional[Union[str, bytes, Dict, BinaryIO]] = None - json: Optional[Dict] = None - header: Optional[str] = None - filename: str = None - # [[ "key", ["filename", "content", "datatype"], - # ["key", ["filename2", "content", "datatype"]], - # ["key2", [None, "content", "datatype"]],] - files: Optional[List[Union[Tuple[str, Tuple[str, BinaryIO, - Optional[str]]], Tuple[str, Tuple[None, str, None]]]]] = None - - -@dataclass -class HttpDef: - name: str = None - method: str = None - url: str = None - headers: dict = None - query: dict = None - auth: AuthBase = None - payload: Optional[Payload] = None - certificate: Optional[List[str]] = None - p12: Optional[List[str]] = None - output: str = None - allow_insecure = False - session_clear = False - test_script: str = "" - test_script_lang: ScriptType = ScriptType.JAVA_SCRIPT - proxy: Optional[Dict[str, str]] = None - - def get_har(self): - if self.auth: - request = self.get_prepared_request() - # till now url and headers are enough - # in future according that, - # include all params - # not giving inaccurate results - self.auth(request) - # For any other auth, it is bad - if isinstance(self.auth, HTTPBasicAuth): - self.headers[BASEIC_AUTHORIZATION_HEADER] = request.headers.get( - BASEIC_AUTHORIZATION_HEADER) - elif isinstance(self.auth, AWS4Auth): - for header_key, header_value in request.headers.items(): - self.headers[header_key] = header_value - # # har with httpdigest is not possible - # # as auth is set once, request is redirected to 401 - # # for now, ignoring - # # also for har, certificate also has to be ignored - # # har does't support this - # # -------------------------- - # elif isinstance(self.auth, HTTPDigestAuth): - # self.headers[BASEIC_AUTHORIZATION_HEADER] = self.auth.build_digest_header(self.method, self.url) - - target = { - "url": self.url, - "method": self.method, - "query": self.get_query(), - "headers": self.get_headers(), - "payload": self.get_payload(), - } - return target - - def get_prepared_request(self): - prep = PreparedRequest() - prep.prepare_url(self.url, self.query) - prep.prepare_method(self.method) - prep.prepare_headers(self.headers) - payload = self.payload - prep.prepare_body( - data=payload.data, - json=payload.json, - files=payload.files) - prep.prepare_auth(self.auth, self.url) - request_logger.info(f"auth configured is {self.auth}") - # prep.prepare_hooks({"response": self.save_cookie_call_back}) - if payload.header and CONTENT_TYPE not in prep.headers: - # if content-type is provided by header - # we will not wish to update it - prep.headers[CONTENT_TYPE] = payload.header - request_logger.debug(f'request prepared completely {prep}') - return prep - - def get_payload(self): - if not self.payload: - return None - payload = self.payload - return_data = {} - if payload.data: - if isinstance(payload.data, dict): - return_data["mimeType"] = FORM_URLENCODED - return_data["text"] = urlencode( - json_to_urlencoded_array(payload.data)) - else: - return_data["mimeType"] = payload.header or "text/plain" - if isinstance(payload.data, (str, bytes)): - return_data["text"] = payload.data - else: - try: - data_read = payload.data.read() - return_data["text"] = data_read.decode("utf-8") - except BaseException: - return_data["text"] = "file conversion to uft-8 ran into error" - finally: - payload.data.close() - elif payload.json: - return_data["mimeType"] = APPLICATION_JSON - return_data["text"] = json.dumps(payload.json) - elif payload.files: - return_data["mimeType"] = MULTIPART_FORM_INPUT - params = [] - for ( - name, - (multipart_filename, - multipart_content, - mimetype)) in payload.files: - content = multipart_content - if isinstance(content, IOBase): - multipart_filename = multipart_content.name - content = None - params.append({ - "name": name, - "fileName": multipart_filename, - "value": content, - "contentType": mimetype - }) - return_data["params"] = params - return return_data - - def get_query(self): - return [{"name": key, "value": value} for key, values in self.query.items() for value in - values] if self.query else [] - - def get_headers(self): - return [{"name": key, "value": value} - for key, value in self.headers.items()] if self.headers else [] - - def get_http_from_req(self): - data = None - datajson = None - file = None - json_payload = None - fileswrap = None - type = None - payload = None - if self.payload: - if self.payload.filename: - file = self.payload.filename - elif isinstance(self.payload.data, str): - data = [TripleOrDouble(str=self.payload.data)] - elif isinstance(self.payload.data, dict): - datajson = self.payload.data - elif self.payload.json: - json_payload = self.payload.json - elif self.payload.files: - fileswrap = FilesWrap([]) - for filekey, multipartdata in self.payload.files: - fileswrap.files.append( - MultiPartFile( - name=filekey, - path=multipartdata[1].name if multipartdata[0] else multipartdata[1], - type=multipartdata[2] if len(multipartdata) > 2 else None)) - - payload = ParsePayload( - data=data, - datajson=datajson, - file=file, - json=json_payload, - fileswrap=fileswrap, - type=type) - - query_lines = [] - if self.query: - for key, values in self.query.items(): - for value in values: - query_lines.append( - Line( - header=None, - query=Query( - key=key, - value=value))) - auth_wrap = None - if self.auth: - if isinstance(self.auth, HTTPBasicAuth): - auth_wrap = AuthWrap( - basic_auth=BasicAuth( - self.auth.username, - self.auth.password)) - elif isinstance(self.auth, HTTPDigestAuth): - auth_wrap = AuthWrap( - digest_auth=DigestAuth( - self.auth.username, - self.auth.password)) - elif isinstance(self.auth, HttpNtlmAuth): - auth_wrap = AuthWrap( - ntlm_auth=NtlmAuthWrap( - self.auth.username, - self.auth.password)) - elif RequestsHawkAuth and isinstance(self.auth, RequestsHawkAuth): - hawk_id = self.auth.credentials['id'] - hawk_key = self.auth.credentials['key'] - hawk_algorithm = self.auth.credentials['algorithm'] - auth_wrap = AuthWrap( - hawk_auth=HawkAuth( - hawk_id, hawk_key, hawk_algorithm)) - elif isinstance(self.auth, AWS4Auth): - aws_auth: AWS4Auth = self.auth - auth_wrap = AuthWrap( - aws_auth=AwsAuthWrap( - aws_auth.access_id, - aws_auth.signing_key.secret_key, - aws_auth.service, - aws_auth.region)) - elif isinstance(self.auth, AzureAuth): - auth_wrap = AuthWrap( - azure_auth=self.auth.azure_auth_wrap) - certificate = None - if self.certificate: - certificate = Certificate(*self.certificate) - elif self.p12: - certificate = P12Certificate(*self.p12) - extra_args = [] - if self.session_clear: - extra_args.append(ExtraArg(clear="@clear")) - if self.allow_insecure: - extra_args.append(ExtraArg(insecure="@insecure")) - header_lines = [] - if self.headers: - for key, value in self.headers.items(): - header_lines.append( - Line( - header=Header( - key=key, - value=value), - query=None)) - test_script = TestScript(self.test_script) - test_script.lang = self.test_script_lang - return Http( - namewrap=NameWrap( - self.name), - extra_args=extra_args, - urlwrap=UrlWrap( - url=self.url, - method=self.method), - lines=header_lines + - query_lines, - payload=payload, - certificate=certificate, - output=None, - authwrap=auth_wrap, - description=None, - script_wrap=test_script) - - -@dataclass -class Property: - text: List = field(default_factory=list()) - key: Union[str, None] = None - value: Union[str, None] = None class BaseModelProcessor: @@ -769,6 +444,10 @@ def load_extra_flags(self): self.httpdef.session_clear = True elif flag.insecure: self.httpdef.allow_insecure = True + + for current_flag in self.http.extra_args: + if current_flag.no_parent_script: + self.httpdef.no_parent_script = True def load_url(self): request_logger.debug( @@ -1111,6 +790,12 @@ def load_def(self): if self._loaded: return self.httpdef.name = self.args.target or '1' + self.load_extra_flags() + self.load_test_script() + # run prerequest script + # as it will set some variables + self.run_prerequest_script() + self.load_method() self.load_url() self.load_headers() @@ -1119,14 +804,24 @@ def load_def(self): self.load_auth() self.load_proxy() self.load_certificate() - self.load_test_script() - self.load_extra_flags() self.load_output() self._loaded = True + self.script_execution.pre_request_script() + + def run_prerequest_script(self): + execution_cls = js3py.ScriptExecutionJs if self.httpdef.test_script_lang == ScriptType.JAVA_SCRIPT else js3py.ScriptExecutionPython + self.script_execution = execution_cls(self.httpdef, self.property_util) + self.script_execution.init_request_script() + for key,value in self.script_execution.client.properties.updated.items(): + self.property_util.add_command_property(key, value) + def load_test_script(self): self.httpdef.test_script = "" - script_wrap: TestScript = self.get_current_or_base("script_wrap") + if self.httpdef.no_parent_script: + script_wrap: TestScript = self.http.script_wrap + else: + script_wrap: TestScript = self.get_current_or_base("script_wrap") if script_wrap and script_wrap.script: script = script_wrap.script[4:-2] self.httpdef.test_script = script.strip() diff --git a/dothttp/__version__.py b/dothttp/__version__.py index c3f5233..af529a2 100644 --- a/dothttp/__version__.py +++ b/dothttp/__version__.py @@ -1 +1 @@ -__version__ = '0.0.43a5' +__version__ = '0.0.43a6' diff --git a/dothttp/computed.py b/dothttp/computed.py new file mode 100644 index 0000000..91683cf --- /dev/null +++ b/dothttp/computed.py @@ -0,0 +1,307 @@ +from dataclasses import dataclass, field +from io import IOBase +from typing import Union, List, Optional, Dict, Tuple, BinaryIO +from urllib.parse import urlencode + +from requests import PreparedRequest +from requests.auth import HTTPBasicAuth, HTTPDigestAuth, AuthBase + +from .parse_models import * +from .parse_models import Payload as ParsePayload +from .constants import * +from .utils import json_to_urlencoded_array, APPLICATION_JSON + +@dataclass +class Config: + curl: bool + property_file: Union[str, None] + properties: List[str] + env: list + debug: bool + file: str + info: bool + no_cookie: bool + format: bool + stdout: bool = False + experimental: bool = False + target: str = field(default_factory=lambda: '1') + content: str = None + + +@dataclass +class Payload: + data: Optional[Union[str, bytes, Dict, BinaryIO]] = None + json: Optional[Dict] = None + header: Optional[str] = None + filename: str = None + # [[ "key", ["filename", "content", "datatype"], + # ["key", ["filename2", "content", "datatype"]], + # ["key2", [None, "content", "datatype"]],] + files: Optional[List[Union[Tuple[str, Tuple[str, BinaryIO, + Optional[str]]], Tuple[str, Tuple[None, str, None]]]]] = None + def set_data(self, data): + self.data = data + + def set_json_data(self, json_data): + self.json = json_data + + def set_files(self, json_files): + self.files = json_files + + def set_content_type(self, content_type): + self.header = content_type + + def set_file_payload(self, file_name): + self.filename = file_name + + + + +@dataclass +class Property: + text: List = field(default_factory=list()) + key: Union[str, None] = None + value: Union[str, None] = None + + +@dataclass +class HttpDef: + name: str = None + method: str = None + url: str = None + headers: dict = None + query: dict = None + auth: AuthBase = None + payload: Optional[Payload] = None + certificate: Optional[List[str]] = None + p12: Optional[List[str]] = None + output: str = None + allow_insecure = False + session_clear = False + no_parent_script = False + test_script: str = "" + test_script_lang: ScriptType = ScriptType.JAVA_SCRIPT + proxy: Optional[Dict[str, str]] = None + + def get_har(self): + if self.auth: + request = self.get_prepared_request() + # till now url and headers are enough + # in future according that, + # include all params + # not giving inaccurate results + self.auth(request) + # For any other auth, it is bad + if isinstance(self.auth, HTTPBasicAuth): + self.headers[BASEIC_AUTHORIZATION_HEADER] = request.headers.get( + BASEIC_AUTHORIZATION_HEADER) + elif isinstance(self.auth, AWS4Auth): + for header_key, header_value in request.headers.items(): + self.headers[header_key] = header_value + # # har with httpdigest is not possible + # # as auth is set once, request is redirected to 401 + # # for now, ignoring + # # also for har, certificate also has to be ignored + # # har does't support this + # # -------------------------- + # elif isinstance(self.auth, HTTPDigestAuth): + # self.headers[BASEIC_AUTHORIZATION_HEADER] = self.auth.build_digest_header(self.method, self.url) + + target = { + "url": self.url, + "method": self.method, + "query": self.get_query(), + "headers": self.get_headers(), + "payload": self.get_payload(), + } + return target + + def get_prepared_request(self): + prep = PreparedRequest() + prep.prepare_url(self.url, self.query) + prep.prepare_method(self.method) + prep.prepare_headers(self.headers) + payload = self.payload + prep.prepare_body( + data=payload.data, + json=payload.json, + files=payload.files) + prep.prepare_auth(self.auth, self.url) + request_logger.info(f"auth configured is {self.auth}") + # prep.prepare_hooks({"response": self.save_cookie_call_back}) + if payload.header and CONTENT_TYPE not in prep.headers: + # if content-type is provided by header + # we will not wish to update it + prep.headers[CONTENT_TYPE] = payload.header + request_logger.debug(f'request prepared completely {prep}') + return prep + + def get_payload(self): + if not self.payload: + return None + payload = self.payload + return_data = {} + if payload.data: + if isinstance(payload.data, dict): + return_data["mimeType"] = FORM_URLENCODED + return_data["text"] = urlencode( + json_to_urlencoded_array(payload.data)) + else: + return_data["mimeType"] = payload.header or "text/plain" + if isinstance(payload.data, (str, bytes)): + return_data["text"] = payload.data + else: + try: + data_read = payload.data.read() + return_data["text"] = data_read.decode("utf-8") + except BaseException: + return_data["text"] = "file conversion to uft-8 ran into error" + finally: + payload.data.close() + elif payload.json: + return_data["mimeType"] = APPLICATION_JSON + return_data["text"] = json.dumps(payload.json) + elif payload.files: + return_data["mimeType"] = MULTIPART_FORM_INPUT + params = [] + for ( + name, + (multipart_filename, + multipart_content, + mimetype)) in payload.files: + content = multipart_content + if isinstance(content, IOBase): + multipart_filename = multipart_content.name + content = None + params.append({ + "name": name, + "fileName": multipart_filename, + "value": content, + "contentType": mimetype + }) + return_data["params"] = params + return return_data + + def get_query(self): + return [{"name": key, "value": value} for key, values in self.query.items() for value in + values] if self.query else [] + + def get_headers(self): + return [{"name": key, "value": value} + for key, value in self.headers.items()] if self.headers else [] + + def get_http_from_req(self): + data = None + datajson = None + file = None + json_payload = None + fileswrap = None + type = None + payload = None + if self.payload: + if self.payload.filename: + file = self.payload.filename + elif isinstance(self.payload.data, str): + data = [TripleOrDouble(str=self.payload.data)] + elif isinstance(self.payload.data, dict): + datajson = self.payload.data + elif self.payload.json: + json_payload = self.payload.json + elif self.payload.files: + fileswrap = FilesWrap([]) + for filekey, multipartdata in self.payload.files: + fileswrap.files.append( + MultiPartFile( + name=filekey, + path=multipartdata[1].name if multipartdata[0] else multipartdata[1], + type=multipartdata[2] if len(multipartdata) > 2 else None)) + + payload = ParsePayload( + data=data, + datajson=datajson, + file=file, + json=json_payload, + fileswrap=fileswrap, + type=type) + + query_lines = [] + if self.query: + for key, values in self.query.items(): + for value in values: + query_lines.append( + Line( + header=None, + query=Query( + key=key, + value=value))) + auth_wrap = None + if self.auth: + if isinstance(self.auth, HTTPBasicAuth): + auth_wrap = AuthWrap( + basic_auth=BasicAuth( + self.auth.username, + self.auth.password)) + elif isinstance(self.auth, HTTPDigestAuth): + auth_wrap = AuthWrap( + digest_auth=DigestAuth( + self.auth.username, + self.auth.password)) + elif isinstance(self.auth, HttpNtlmAuth): + auth_wrap = AuthWrap( + ntlm_auth=NtlmAuthWrap( + self.auth.username, + self.auth.password)) + elif RequestsHawkAuth and isinstance(self.auth, RequestsHawkAuth): + hawk_id = self.auth.credentials['id'] + hawk_key = self.auth.credentials['key'] + hawk_algorithm = self.auth.credentials['algorithm'] + auth_wrap = AuthWrap( + hawk_auth=HawkAuth( + hawk_id, hawk_key, hawk_algorithm)) + elif isinstance(self.auth, AWS4Auth): + aws_auth: AWS4Auth = self.auth + auth_wrap = AuthWrap( + aws_auth=AwsAuthWrap( + aws_auth.access_id, + aws_auth.signing_key.secret_key, + aws_auth.service, + aws_auth.region)) + elif isinstance(self.auth, AzureAuth): + auth_wrap = AuthWrap( + azure_auth=self.auth.azure_auth_wrap) + certificate = None + if self.certificate: + certificate = Certificate(*self.certificate) + elif self.p12: + certificate = P12Certificate(*self.p12) + extra_args = [] + if self.session_clear: + extra_args.append(ExtraArg(clear="@clear")) + if self.allow_insecure: + extra_args.append(ExtraArg(insecure="@insecure")) + header_lines = [] + if self.headers: + for key, value in self.headers.items(): + header_lines.append( + Line( + header=Header( + key=key, + value=value), + query=None)) + test_script = TestScript(self.test_script) + test_script.lang = self.test_script_lang + return Http( + namewrap=NameWrap( + self.name), + extra_args=extra_args, + urlwrap=UrlWrap( + url=self.url, + method=self.method), + lines=header_lines + + query_lines, + payload=payload, + certificate=certificate, + output=None, + authwrap=auth_wrap, + description=None, + script_wrap=test_script) diff --git a/dothttp/constants.py b/dothttp/constants.py new file mode 100644 index 0000000..09a58d2 --- /dev/null +++ b/dothttp/constants.py @@ -0,0 +1,50 @@ +import logging + +MIME_TYPE_JSON = "application/json" +FORM_URLENCODED = "application/x-www-form-urlencoded" +MULTIPART_FORM_INPUT = "multipart/form-data" +TEXT_PLAIN = "text/plain" +CONTENT_TYPE = 'content-type' + +UNIX_SOCKET_SCHEME = "http+unix" + +BASEIC_AUTHORIZATION_HEADER = "Authorization" + + +base_logger = logging.getLogger("dothttp") +request_logger = logging.getLogger("request") +curl_logger = logging.getLogger("curl") + + +try: + from requests_aws4auth import AWS4Auth +except ImportError: + AWS4Auth = None +try: + from requests_ntlm import HttpNtlmAuth +except ImportError: + HttpNtlmAuth = None + +try: + import jstyleson as json + from jsonschema import validate +except ImportError: + import json + validate = None + +try: + from .azure_auth import AzureAuth +except: + # this is for dothttp-wasm, where msal most likely not installed + AzureAuth = None + +try: + import magic +except ImportError: + magic = None + +try: + from requests_hawk import HawkAuth as RequestsHawkAuth +except BaseException: + RequestsHawkAuth = None + diff --git a/dothttp/http.tx b/dothttp/http.tx index 0f49316..f53f537 100644 --- a/dothttp/http.tx +++ b/dothttp/http.tx @@ -118,7 +118,9 @@ AZURECLIAUTH: EXTRA_ARG: // there can be more - clear=CLEAR_SESSION | insecure=INSECURE + clear=CLEAR_SESSION + | insecure=INSECURE + | no_parent_script=CLEAR_PARENT_SCRIPT ; NAMED_ARG: @@ -130,6 +132,10 @@ CLEAR_SESSION: '@clear' ; +CLEAR_PARENT_SCRIPT: + '@no_parent_script' +; + INSECURE: "@insecure" ; diff --git a/dothttp/js3py.py b/dothttp/js3py.py index 44a2c29..97cb2de 100644 --- a/dothttp/js3py.py +++ b/dothttp/js3py.py @@ -10,6 +10,8 @@ import unittest import uuid import urllib +import json +import yaml import urllib.parse import cryptography from cryptography import * @@ -27,9 +29,7 @@ from operator import getitem from faker import Faker -from dothttp.exceptions import DotHttpException, PreRequestScriptException, ScriptException - - +from .exceptions import DotHttpException, PreRequestScriptException, ScriptException from .property_util import PropertyProvider from . import MIME_TYPE_JSON, HttpDef, request_logger from .utils import get_real_file_path @@ -63,6 +63,8 @@ def write_guard(x): 'base64': base64, 'urllib': urllib, 'open': open, + 'json': json, + 'yaml': yaml, 'cryptography': cryptography } allowed_global.update(safe_globals) @@ -157,14 +159,22 @@ def clear_all(self, key): class Client: request: HttpDef properties: Properties + env_properties: Properties + infile_properties: Properties response: Response = None class ScriptExecutionEnvironmentBase: def __init__(self, httpdef: HttpDef, prop: PropertyProvider) -> None: self.client = Client( - request=httpdef, properties=Properties( - prop.get_all_properties_variables())) + request=httpdef, + properties=Properties(prop.get_all_properties_variables()), + infile_properties={ key:value.value for key, value in prop.infile_properties.items()}, + env_properties=dict(prop.env_properties) + ) + + def _init_request_script(self) -> None: + pass def _pre_request_script(self) -> None: pass @@ -172,6 +182,13 @@ def _pre_request_script(self) -> None: def _execute_test_script(self, resp: Response) -> ScriptResult: raise NotImplementedError() + def init_request_script(self): + try: + self._init_request_script() + except Exception as exc: + request_logger.error("unknown exception happened", exc_info=True) + raise PreRequestScriptException(payload=str(exc)) + def pre_request_script(self): try: self._pre_request_script() @@ -261,6 +278,13 @@ def __init__(self, httpdef: HttpDef, prop: PropertyProvider) -> None: except Exception as exc: raise ScriptException(payload=str(exc)) + def _init_request_script(self) -> None: + # just for variables initialization + for key, func in self.local.items(): + if (key.startswith('init')) and isinstance( + func, types.FunctionType): + func() + def _pre_request_script(self) -> None: for key, func in self.local.items(): if (key.startswith('pre')) and isinstance( diff --git a/dothttp/parse_models.py b/dothttp/parse_models.py index c0aa6ac..3e87d9b 100644 --- a/dothttp/parse_models.py +++ b/dothttp/parse_models.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from typing import List, Optional, Union -from dothttp import DotHttpException +from .exceptions import DotHttpException @dataclass class NameWrap: @@ -197,6 +197,8 @@ class ExtraArg: clear: Optional[str] = '' # allows insecure insecure: Optional[str] = '' + # ignore parent script + no_parent_script: Optional[str] = '' @dataclass diff --git a/dothttp/request_base.py b/dothttp/request_base.py index 40eb092..9496665 100644 --- a/dothttp/request_base.py +++ b/dothttp/request_base.py @@ -419,9 +419,6 @@ class RequestCompiler(RequestBase): def run(self): self.load_def() - execution_cls = js3py.ScriptExecutionJs if self.httpdef.test_script_lang == ScriptType.JAVA_SCRIPT else js3py.ScriptExecutionPython - script_execution = execution_cls(self.httpdef, self.property_util) - script_execution.pre_request_script() resp = self.get_response() self.print_req_info(resp.request) for hist_resp in resp.history: @@ -437,7 +434,7 @@ def run(self): self.print_req_info(resp, '<') self.write_to_output(resp) request_logger.debug(f'request executed completely') - script_result = script_execution.execute_test_script(resp=resp) + script_result = self.script_execution.execute_test_script(resp=resp) self.print_script_result(script_result) return resp diff --git a/examples/example.http b/examples/example.http index bfcef54..07ae959 100644 --- a/examples/example.http +++ b/examples/example.http @@ -1,7 +1,7 @@ @name("fetch 100 users, skip first 50") GET https://req.dothttp.dev/user "Authorization": "Basic dXNlcm5hbWU6cGFzc3dvcmQ=" -"content-type": "hai" +"content-type": "application/json" ? ("fetch", "100") # ? ("skip", "50") ? projection, name diff --git a/examples/init_script.http b/examples/init_script.http new file mode 100644 index 0000000..af1a6bf --- /dev/null +++ b/examples/init_script.http @@ -0,0 +1,24 @@ +# this request reads file, base64 encodes and sends request +@name("init and pre request example"): "example" +GET "/ram" + +@name("uses parent request but not script"): "example" +@no_parent_script +GET "/laxman" + + + +@name("example") +GET https://req.dothttp.dev/user + +text('hai') +> {% + +def init_set_value(): + with open('/tmp/haha', 'r') as f: + client.properties.set("ram", base64.b64encode(f.read().encode()).decode()) + +def pre_setup(): + client.request.payload.set_data(client.properties.get("ram", "not available")) + +%} python diff --git a/requirements.txt b/requirements.txt index dc0b250..357091c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,12 +2,12 @@ jsonschema==4.21.1 jstyleson==0.0.2 textx==4.0.1 js2py==0.74 -requests_pkcs12==1.22 +requests_pkcs12==1.24 parsys-requests-unixsocket==0.3.1 requests-aws4auth==1.2.3 requests_ntlm==1.2.0 RestrictedPython==7.0 -Faker==22.6.0 +Faker==23.1.0 requests-hawk==1.2.1 msal==1.26.0 PyYaml==6.0.1 diff --git a/test_requirements.txt b/test_requirements.txt index dad1318..aa93477 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,2 +1,2 @@ -waitress==2.1.1 +waitress==3.0.0 flask==3.0.2