Rest api

Module to handle REST API operations.

RESTApiException

Bases: requests.RequestException

Class representing any possible REST API Exception.

Source code in mkdocs/lakehouse_engine/packages/utils/rest_api.py

class RESTApiException(requests.RequestException):
    """Class representing any possible REST API Exception."""

    def __init__(self, message: str) -> None:
        """Construct RESTApiException instances.

        Args:
            message: message to display on exception event.
        """
        super().__init__(message)

__init__(message)

Construct RESTApiException instances.

Parameters:

Name	Type	Description	Default
message	str	message to display on exception event.	required

Source code in mkdocs/lakehouse_engine/packages/utils/rest_api.py

def __init__(self, message: str) -> None:
    """Construct RESTApiException instances.

    Args:
        message: message to display on exception event.
    """
    super().__init__(message)

RestMethods

Bases: Enum

Methods for REST API calls.

Source code in mkdocs/lakehouse_engine/packages/utils/rest_api.py

class RestMethods(Enum):
    """Methods for REST API calls."""

    POST = "POST"
    PUT = "PUT"
    ALLOWED_METHODS = ["POST", "PUT"]

RestStatusCodes

Bases: Enum

REST Status Code.

Source code in mkdocs/lakehouse_engine/packages/utils/rest_api.py

class RestStatusCodes(Enum):
    """REST Status Code."""

    RETRY_STATUS_CODES = [429, 500, 502, 503, 504]
    OK_STATUS_CODES = [200]

execute_api_request(method, url, headers=None, basic_auth_dict=None, json=None, files=None, sleep_seconds=0.2)

Execute a REST API request.

Parameters:

Name	Type	Description	Default
method	str	REST method (e.g., POST or PUT).	required
url	str	url of the api.	required
headers	dict	request headers.	None
basic_auth_dict	dict	basic http authentication details (e.g., {"username": "x", "password": "y"}).	None
json	dict	json payload to send in the request.	None
files	dict	files payload to send in the request.	None
sleep_seconds	float	for how many seconds to sleep to avoid error 429.	0.2

Returns:

Type	Description
requests.Response	response from the HTTP request.

Source code in mkdocs/lakehouse_engine/packages/utils/rest_api.py

def execute_api_request(
    method: str,
    url: str,
    headers: dict = None,
    basic_auth_dict: dict = None,
    json: dict = None,
    files: dict = None,
    sleep_seconds: float = 0.2,
) -> requests.Response:
    """Execute a REST API request.

    Args:
        method: REST method (e.g., POST or PUT).
        url: url of the api.
        headers: request headers.
        basic_auth_dict: basic http authentication details
            (e.g., {"username": "x", "password": "y"}).
        json: json payload to send in the request.
        files: files payload to send in the request.
        sleep_seconds: for how many seconds to sleep to avoid error 429.

    Returns:
        response from the HTTP request.
    """
    basic_auth: requests.auth.HTTPBasicAuth = None
    if basic_auth_dict:
        basic_auth = get_basic_auth(
            basic_auth_dict["username"], basic_auth_dict["password"]
        )

    return get_configured_session(sleep_seconds=sleep_seconds).request(
        method=method,
        url=url,
        headers=headers,
        auth=basic_auth,
        json=json,
        files=files,
    )

get_basic_auth(username, password)

Get the basic authentication object to authenticate REST requests.

Parameters:

Name	Type	Description	Default
username	str	username.	required
password	str	password.	required

Returns:

Type	Description
requests.auth.HTTPBasicAuth	requests.auth.HTTPBasicAuth: the HTTPBasicAuth object.

Source code in mkdocs/lakehouse_engine/packages/utils/rest_api.py

def get_basic_auth(username: str, password: str) -> requests.auth.HTTPBasicAuth:
    """Get the basic authentication object to authenticate REST requests.

    Args:
        username: username.
        password: password.

    Returns:
        requests.auth.HTTPBasicAuth: the HTTPBasicAuth object.
    """
    return requests.auth.HTTPBasicAuth(username, password)

get_configured_session(sleep_seconds=0.2, total_retries=5, backoff_factor=2, retry_status_codes=None, allowed_methods=None, protocol='https://')

Get a configured requests Session with exponential backoff.

Parameters:

Name	Type	Description	Default
sleep_seconds	float	seconds to sleep before each request to avoid rate limits.	0.2
total_retries	int	number of times to retry.	5
backoff_factor	int	factor for the exponential backoff.	2
retry_status_codes	list	list of status code that triggers a retry.	None
allowed_methods	list	http methods that are allowed for retry.	None
protocol	str	http:// or https://.	'https://'

Returns requests.Session: the configured session.

Source code in mkdocs/lakehouse_engine/packages/utils/rest_api.py

def get_configured_session(
    sleep_seconds: float = 0.2,
    total_retries: int = 5,
    backoff_factor: int = 2,
    retry_status_codes: list = None,
    allowed_methods: list = None,
    protocol: str = "https://",
) -> requests.Session:
    """Get a configured requests Session with exponential backoff.

    Args:
        sleep_seconds: seconds to sleep before each request to avoid rate limits.
        total_retries: number of times to retry.
        backoff_factor: factor for the exponential backoff.
        retry_status_codes: list of status code that triggers a retry.
        allowed_methods: http methods that are allowed for retry.
        protocol: http:// or https://.

    Returns
        requests.Session: the configured session.
    """
    retry_status_codes = (
        retry_status_codes
        if retry_status_codes
        else RestStatusCodes.RETRY_STATUS_CODES.value
    )
    allowed_methods = (
        allowed_methods if allowed_methods else RestMethods.ALLOWED_METHODS.value
    )
    time.sleep(sleep_seconds)
    session = requests.Session()
    retries = Retry(
        total=total_retries,
        backoff_factor=backoff_factor,
        status_forcelist=retry_status_codes,
        allowed_methods=allowed_methods,
    )
    session.mount(protocol, HTTPAdapter(max_retries=retries))
    return session