Skip to content

File manager

Module for abstract representation of a file manager system.

FileManager

Bases: ABC

Abstract file manager class.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
class FileManager(ABC):  # noqa: B024
    """Abstract file manager class."""

    def __init__(self, configs: dict):
        """Construct FileManager algorithm instances.

        Args:
            configs: configurations for the FileManager algorithm.
        """
        self.configs = configs
        self.function = self.configs["function"]

    @abstractmethod
    def delete_objects(self) -> None:
        """Delete objects and 'directories'.

        If dry_run is set to True the function will print a dict with all the
        paths that would be deleted based on the given keys.
        """
        pass

    @abstractmethod
    def copy_objects(self) -> None:
        """Copies objects and 'directories'.

        If dry_run is set to True the function will print a dict with all the
        paths that would be copied based on the given keys.
        """
        pass

    @abstractmethod
    def move_objects(self) -> None:
        """Moves objects and 'directories'.

        If dry_run is set to True the function will print a dict with all the
        paths that would be moved based on the given keys.
        """
        pass

__init__(configs)

Construct FileManager algorithm instances.

Parameters:

Name Type Description Default
configs dict

configurations for the FileManager algorithm.

 required
Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
13
14
15
16
17
18
19
20
def __init__(self, configs: dict):
    """Construct FileManager algorithm instances.

    Args:
        configs: configurations for the FileManager algorithm.
    """
    self.configs = configs
    self.function = self.configs["function"]

copy_objects() abstractmethod

Copies objects and 'directories'.

If dry_run is set to True the function will print a dict with all the paths that would be copied based on the given keys.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
31
32
33
34
35
36
37
38
@abstractmethod
def copy_objects(self) -> None:
    """Copies objects and 'directories'.

    If dry_run is set to True the function will print a dict with all the
    paths that would be copied based on the given keys.
    """
    pass

delete_objects() abstractmethod

Delete objects and 'directories'.

If dry_run is set to True the function will print a dict with all the paths that would be deleted based on the given keys.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
22
23
24
25
26
27
28
29
@abstractmethod
def delete_objects(self) -> None:
    """Delete objects and 'directories'.

    If dry_run is set to True the function will print a dict with all the
    paths that would be deleted based on the given keys.
    """
    pass

move_objects() abstractmethod

Moves objects and 'directories'.

If dry_run is set to True the function will print a dict with all the paths that would be moved based on the given keys.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
40
41
42
43
44
45
46
47
@abstractmethod
def move_objects(self) -> None:
    """Moves objects and 'directories'.

    If dry_run is set to True the function will print a dict with all the
    paths that would be moved based on the given keys.
    """
    pass

FileManagerFactory

Bases: ABC

Class for file manager factory.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
class FileManagerFactory(ABC):  # noqa: B024
    """Class for file manager factory."""

    @staticmethod
    def execute_function(configs: dict) -> Any:
        """Get a specific File Manager and function to execute."""
        from lakehouse_engine.core.dbfs_file_manager import DBFSFileManager
        from lakehouse_engine.core.s3_file_manager import S3FileManager

        disable_dbfs_retry = (
            configs["disable_dbfs_retry"]
            if "disable_dbfs_retry" in configs.keys()
            else False
        )

        if disable_dbfs_retry:
            S3FileManager(configs).get_function()
        elif FileStorageFunctions.is_boto3_configured():
            try:
                S3FileManager(configs).get_function()
            except (ValueError, NotImplementedError, RestoreTypeNotFoundException):
                raise
            except Exception:
                DBFSFileManager(configs).get_function()
        else:
            DBFSFileManager(configs).get_function()

execute_function(configs) staticmethod

Get a specific File Manager and function to execute.

Source code in mkdocs/lakehouse_engine/packages/core/file_manager.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
@staticmethod
def execute_function(configs: dict) -> Any:
    """Get a specific File Manager and function to execute."""
    from lakehouse_engine.core.dbfs_file_manager import DBFSFileManager
    from lakehouse_engine.core.s3_file_manager import S3FileManager

    disable_dbfs_retry = (
        configs["disable_dbfs_retry"]
        if "disable_dbfs_retry" in configs.keys()
        else False
    )

    if disable_dbfs_retry:
        S3FileManager(configs).get_function()
    elif FileStorageFunctions.is_boto3_configured():
        try:
            S3FileManager(configs).get_function()
        except (ValueError, NotImplementedError, RestoreTypeNotFoundException):
            raise
        except Exception:
            DBFSFileManager(configs).get_function()
    else:
        DBFSFileManager(configs).get_function()