Skip to content

File manager

Module for abstract representation of a file manager system.

FileManager

Bases: ABC

Macro Rendering Error

File: reference/packages/core/file_manager.md

ModuleNotFoundError: No module named 'pyspark'

Traceback (most recent call last):
  File "/app/tmp_os/lakehouse-engine/.venv/lib/python3.12/site-packages/mkdocs_macros/plugin.py", line 703, in render
    return md_template.render(**page_variables)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/app/tmp_os/lakehouse-engine/.venv/lib/python3.12/site-packages/jinja2/environment.py", line 1295, in render
    self.environment.handle_exception()
  File "/app/tmp_os/lakehouse-engine/.venv/lib/python3.12/site-packages/jinja2/environment.py", line 942, in handle_exception
    raise rewrite_traceback_stack(source=source)
  File "<template>", line 3, in top-level template code
  File "/app/tmp_os/lakehouse-engine/cicd/code_doc/mkdocs_macros.py", line 194, in get_file_manager_operations
    from lakehouse_engine.core.definitions import FILE_MANAGER_OPERATIONS
  File "/app/tmp_os/lakehouse-engine/lakehouse_engine/core/definitions.py", line 10, in <module>
    from pyspark.sql import DataFrame
ModuleNotFoundError: No module named 'pyspark'
Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
class FileManager(ABC):
    """Abstract file manager class.

    {{ get_file_manager_operations() }}
    """

    def __init__(self, configs: dict) -> None:
        """Construct FileManager algorithm instances.

        Args:
            configs: configurations for the FileManager algorithm.
        """
        self.configs = configs
        self.function = self.configs["function"]

    @abstractmethod
    def delete_objects(self) -> None:
        """Delete objects and 'directories'.

        If dry_run is set to True the function will print a dict with all the
        paths that would be deleted based on the given keys.
        """

    @abstractmethod
    def copy_objects(self) -> None:
        """Copies objects and 'directories'.

        If dry_run is set to True the function will print a dict with all the
        paths that would be copied based on the given keys.
        """

    @abstractmethod
    def move_objects(self) -> None:
        """Moves objects and 'directories'.

        If dry_run is set to True the function will print a dict with all the
        paths that would be moved based on the given keys.
        """

__init__(configs)

Construct FileManager algorithm instances.

Parameters:

Name Type Description Default
configs dict

configurations for the FileManager algorithm.

 required
Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
16
17
18
19
20
21
22
23
def __init__(self, configs: dict) -> None:
    """Construct FileManager algorithm instances.

    Args:
        configs: configurations for the FileManager algorithm.
    """
    self.configs = configs
    self.function = self.configs["function"]

copy_objects() abstractmethod

Copies objects and 'directories'.

If dry_run is set to True the function will print a dict with all the paths that would be copied based on the given keys.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
33
34
35
36
37
38
39
@abstractmethod
def copy_objects(self) -> None:
    """Copies objects and 'directories'.

    If dry_run is set to True the function will print a dict with all the
    paths that would be copied based on the given keys.
    """

delete_objects() abstractmethod

Delete objects and 'directories'.

If dry_run is set to True the function will print a dict with all the paths that would be deleted based on the given keys.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
25
26
27
28
29
30
31
@abstractmethod
def delete_objects(self) -> None:
    """Delete objects and 'directories'.

    If dry_run is set to True the function will print a dict with all the
    paths that would be deleted based on the given keys.
    """

move_objects() abstractmethod

Moves objects and 'directories'.

If dry_run is set to True the function will print a dict with all the paths that would be moved based on the given keys.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
41
42
43
44
45
46
47
@abstractmethod
def move_objects(self) -> None:
    """Moves objects and 'directories'.

    If dry_run is set to True the function will print a dict with all the
    paths that would be moved based on the given keys.
    """

FileManagerFactory

Bases: ABC

Class for file manager factory.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
class FileManagerFactory(ABC):
    """Class for file manager factory."""

    @staticmethod
    def execute_function(configs: dict) -> Any:
        """Get a specific File Manager and function to execute."""
        from lakehouse_engine.core.dbfs_file_manager import DBFSFileManager
        from lakehouse_engine.core.s3_file_manager import S3FileManager

        disable_dbfs_retry = configs.get("disable_dbfs_retry", False)

        if disable_dbfs_retry:
            S3FileManager(configs).get_function()
        elif FileStorageFunctions.is_boto3_configured():
            try:
                S3FileManager(configs).get_function()
            except (ValueError, NotImplementedError, RestoreTypeNotFoundException):
                raise
            except Exception:  # noqa: BLE001
                DBFSFileManager(configs).get_function()
        else:
            DBFSFileManager(configs).get_function()

execute_function(configs) staticmethod

Get a specific File Manager and function to execute.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
@staticmethod
def execute_function(configs: dict) -> Any:
    """Get a specific File Manager and function to execute."""
    from lakehouse_engine.core.dbfs_file_manager import DBFSFileManager
    from lakehouse_engine.core.s3_file_manager import S3FileManager

    disable_dbfs_retry = configs.get("disable_dbfs_retry", False)

    if disable_dbfs_retry:
        S3FileManager(configs).get_function()
    elif FileStorageFunctions.is_boto3_configured():
        try:
            S3FileManager(configs).get_function()
        except (ValueError, NotImplementedError, RestoreTypeNotFoundException):
            raise
        except Exception:  # noqa: BLE001
            DBFSFileManager(configs).get_function()
    else:
        DBFSFileManager(configs).get_function()