upload

zero-cost-nas/.eggs/smmap-5.0.0-py3.8.egg/smmap/util.py

@@ -0,0 +1,222 @@
+"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files"""
+import os
+import sys
+
+from mmap import mmap, ACCESS_READ
+from mmap import ALLOCATIONGRANULARITY
+
+__all__ = ["align_to_mmap", "is_64_bit",
+           "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"]
+
+#{ Utilities
+
+
+def align_to_mmap(num, round_up):
+    """
+    Align the given integer number to the closest page offset, which usually is 4096 bytes.
+
+    :param round_up: if True, the next higher multiple of page size is used, otherwise
+        the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0)
+    :return: num rounded to closest page"""
+    res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY
+    if round_up and (res != num):
+        res += ALLOCATIONGRANULARITY
+    # END handle size
+    return res
+
+
+def is_64_bit():
+    """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit"""
+    return sys.maxsize > (1 << 32) - 1
+
+#}END utilities
+
+
+#{ Utility Classes
+
+class MapWindow:
+
+    """Utility type which is used to snap windows towards each other, and to adjust their size"""
+    __slots__ = (
+        'ofs',      # offset into the file in bytes
+        'size'              # size of the window in bytes
+    )
+
+    def __init__(self, offset, size):
+        self.ofs = offset
+        self.size = size
+
+    def __repr__(self):
+        return "MapWindow(%i, %i)" % (self.ofs, self.size)
+
+    @classmethod
+    def from_region(cls, region):
+        """:return: new window from a region"""
+        return cls(region._b, region.size())
+
+    def ofs_end(self):
+        return self.ofs + self.size
+
+    def align(self):
+        """Assures the previous window area is contained in the new one"""
+        nofs = align_to_mmap(self.ofs, 0)
+        self.size += self.ofs - nofs    # keep size constant
+        self.ofs = nofs
+        self.size = align_to_mmap(self.size, 1)
+
+    def extend_left_to(self, window, max_size):
+        """Adjust the offset to start where the given window on our left ends if possible,
+        but don't make yourself larger than max_size.
+        The resize will assure that the new window still contains the old window area"""
+        rofs = self.ofs - window.ofs_end()
+        nsize = rofs + self.size
+        rofs -= nsize - min(nsize, max_size)
+        self.ofs = self.ofs - rofs
+        self.size += rofs
+
+    def extend_right_to(self, window, max_size):
+        """Adjust the size to make our window end where the right window begins, but don't
+        get larger than max_size"""
+        self.size = min(self.size + (window.ofs - self.ofs_end()), max_size)
+
+
+class MapRegion:
+
+    """Defines a mapped region of memory, aligned to pagesizes
+
+    **Note:** deallocates used region automatically on destruction"""
+    __slots__ = [
+        '_b',   # beginning of mapping
+        '_mf',  # mapped memory chunk (as returned by mmap)
+        '_uc',  # total amount of usages
+        '_size',  # cached size of our memory map
+        '__weakref__'
+    ]
+
+    #{ Configuration
+    #} END configuration
+
+    def __init__(self, path_or_fd, ofs, size, flags=0):
+        """Initialize a region, allocate the memory map
+        :param path_or_fd: path to the file to map, or the opened file descriptor
+        :param ofs: **aligned** offset into the file to be mapped
+        :param size: if size is larger then the file on disk, the whole file will be
+            allocated the the size automatically adjusted
+        :param flags: additional flags to be given when opening the file.
+        :raise Exception: if no memory can be allocated"""
+        self._b = ofs
+        self._size = 0
+        self._uc = 0
+
+        if isinstance(path_or_fd, int):
+            fd = path_or_fd
+        else:
+            fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags)
+        # END handle fd
+
+        try:
+            kwargs = dict(access=ACCESS_READ, offset=ofs)
+            corrected_size = size
+            sizeofs = ofs
+
+            # have to correct size, otherwise (instead of the c version) it will
+            # bark that the size is too large ... many extra file accesses because
+            # if this ... argh !
+            actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size)
+            self._mf = mmap(fd, actual_size, **kwargs)
+            # END handle memory mode
+
+            self._size = len(self._mf)
+        finally:
+            if isinstance(path_or_fd, str):
+                os.close(fd)
+            # END only close it if we opened it
+        # END close file handle
+        # We assume the first one to use us keeps us around
+        self.increment_client_count()
+
+    def __repr__(self):
+        return "MapRegion<%i, %i>" % (self._b, self.size())
+
+    #{ Interface
+
+    def buffer(self):
+        """:return: a buffer containing the memory"""
+        return self._mf
+
+    def map(self):
+        """:return: a memory map containing the memory"""
+        return self._mf
+
+    def ofs_begin(self):
+        """:return: absolute byte offset to the first byte of the mapping"""
+        return self._b
+
+    def size(self):
+        """:return: total size of the mapped region in bytes"""
+        return self._size
+
+    def ofs_end(self):
+        """:return: Absolute offset to one byte beyond the mapping into the file"""
+        return self._b + self._size
+
+    def includes_ofs(self, ofs):
+        """:return: True if the given offset can be read in our mapped region"""
+        return self._b <= ofs < self._b + self._size
+
+    def client_count(self):
+        """:return: number of clients currently using this region"""
+        return self._uc
+
+    def increment_client_count(self, ofs = 1):
+        """Adjust the usage count by the given positive or negative offset.
+        If usage count equals 0, we will auto-release our resources
+        :return: True if we released resources, False otherwise. In the latter case, we can still be used"""
+        self._uc += ofs
+        assert self._uc > -1, "Increments must match decrements, usage counter negative: %i" % self._uc
+
+        if self.client_count() == 0:
+            self.release()
+            return True
+        else:
+            return False
+        # end handle release
+
+    def release(self):
+        """Release all resources this instance might hold. Must only be called if there usage_count() is zero"""
+        self._mf.close()
+
+    #} END interface
+
+
+class MapRegionList(list):
+
+    """List of MapRegion instances associating a path with a list of regions."""
+    __slots__ = (
+        '_path_or_fd',  # path or file descriptor which is mapped by all our regions
+        '_file_size'    # total size of the file we map
+    )
+
+    def __new__(cls, path):
+        return super().__new__(cls)
+
+    def __init__(self, path_or_fd):
+        self._path_or_fd = path_or_fd
+        self._file_size = None
+
+    def path_or_fd(self):
+        """:return: path or file descriptor we are attached to"""
+        return self._path_or_fd
+
+    def file_size(self):
+        """:return: size of file we manager"""
+        if self._file_size is None:
+            if isinstance(self._path_or_fd, str):
+                self._file_size = os.stat(self._path_or_fd).st_size
+            else:
+                self._file_size = os.fstat(self._path_or_fd).st_size
+            # END handle path type
+        # END update file size
+        return self._file_size
+
+#} END utility classes