Add support for long paths
* Add more file i/o wrappers in platform_utils to allow using
long paths (length > MAX_PATH) on Windows.
* Paths using the long path syntax ("\\?\" prefix) should never
escape the platform_utils API surface area, so that this
specific syntax is not visible to the rest of the repo code base.
* Forward many calls from os.xxx to platform_utils.xxx in various place
to ensure long paths support, specifically when repo decides to delete
obsolete directories.
* There are more places that need to be converted to support long paths,
this commit is an initial effort to unblock a few common use cases.
* Also, fix remove function to handle directory symlinks
Change-Id: If82ccc408e516e96ff7260be25f8fd2fe3f9571a
diff --git a/platform_utils.py b/platform_utils.py
index a3e9653..b2cc245 100644
--- a/platform_utils.py
+++ b/platform_utils.py
@@ -187,10 +187,10 @@
source = _validate_winpath(source)
link_name = _validate_winpath(link_name)
target = os.path.join(os.path.dirname(link_name), source)
- if os.path.isdir(target):
- platform_utils_win32.create_dirsymlink(source, link_name)
+ if isdir(target):
+ platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name)
else:
- platform_utils_win32.create_filesymlink(source, link_name)
+ platform_utils_win32.create_filesymlink(_makelongpath(source), link_name)
else:
return os.symlink(source, link_name)
@@ -220,9 +220,32 @@
return not drive # "x:" is invalid
-def rmtree(path):
+def _makelongpath(path):
+ """Return the input path normalized to support the Windows long path syntax
+ ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
+ MAX_PATH limit.
+ """
if isWindows():
- shutil.rmtree(path, onerror=handle_rmtree_error)
+ # Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246.
+ if len(path) < 246:
+ return path
+ if path.startswith(u"\\\\?\\"):
+ return path
+ if not os.path.isabs(path):
+ return path
+ # Append prefix and ensure unicode so that the special longpath syntax
+ # is supported by underlying Win32 API calls
+ return u"\\\\?\\" + os.path.normpath(path)
+ else:
+ return path
+
+
+def rmtree(path):
+ """shutil.rmtree(path) wrapper with support for long paths on Windows.
+
+ Availability: Unix, Windows."""
+ if isWindows():
+ shutil.rmtree(_makelongpath(path), onerror=handle_rmtree_error)
else:
shutil.rmtree(path)
@@ -234,15 +257,18 @@
def rename(src, dst):
+ """os.rename(src, dst) wrapper with support for long paths on Windows.
+
+ Availability: Unix, Windows."""
if isWindows():
# On Windows, rename fails if destination exists, see
# https://docs.python.org/2/library/os.html#os.rename
try:
- os.rename(src, dst)
+ os.rename(_makelongpath(src), _makelongpath(dst))
except OSError as e:
if e.errno == errno.EEXIST:
- os.remove(dst)
- os.rename(src, dst)
+ os.remove(_makelongpath(dst))
+ os.rename(_makelongpath(src), _makelongpath(dst))
else:
raise
else:
@@ -250,30 +276,98 @@
def remove(path):
- """Remove (delete) the file path. This is a replacement for os.remove, but
- allows deleting read-only files on Windows.
- """
+ """Remove (delete) the file path. This is a replacement for os.remove that
+ allows deleting read-only files on Windows, with support for long paths and
+ for deleting directory symbolic links.
+
+ Availability: Unix, Windows."""
if isWindows():
+ longpath = _makelongpath(path)
try:
- os.remove(path)
+ os.remove(longpath)
except OSError as e:
if e.errno == errno.EACCES:
- os.chmod(path, stat.S_IWRITE)
- os.remove(path)
+ os.chmod(longpath, stat.S_IWRITE)
+ # Directory symbolic links must be deleted with 'rmdir'.
+ if islink(longpath) and isdir(longpath):
+ os.rmdir(longpath)
+ else:
+ os.remove(longpath)
else:
raise
else:
os.remove(path)
+def walk(top, topdown=True, onerror=None, followlinks=False):
+ """os.walk(path) wrapper with support for long paths on Windows.
+
+ Availability: Windows, Unix.
+ """
+ if isWindows():
+ return _walk_windows_impl(top, topdown, onerror, followlinks)
+ else:
+ return os.walk(top, topdown, onerror, followlinks)
+
+
+def _walk_windows_impl(top, topdown, onerror, followlinks):
+ try:
+ names = listdir(top)
+ except error, err:
+ if onerror is not None:
+ onerror(err)
+ return
+
+ dirs, nondirs = [], []
+ for name in names:
+ if isdir(os.path.join(top, name)):
+ dirs.append(name)
+ else:
+ nondirs.append(name)
+
+ if topdown:
+ yield top, dirs, nondirs
+ for name in dirs:
+ new_path = os.path.join(top, name)
+ if followlinks or not islink(new_path):
+ for x in _walk_windows_impl(new_path, topdown, onerror, followlinks):
+ yield x
+ if not topdown:
+ yield top, dirs, nondirs
+
+
+def listdir(path):
+ """os.listdir(path) wrapper with support for long paths on Windows.
+
+ Availability: Windows, Unix.
+ """
+ return os.listdir(_makelongpath(path))
+
+
+def rmdir(path):
+ """os.rmdir(path) wrapper with support for long paths on Windows.
+
+ Availability: Windows, Unix.
+ """
+ os.rmdir(_makelongpath(path))
+
+
+def isdir(path):
+ """os.path.isdir(path) wrapper with support for long paths on Windows.
+
+ Availability: Windows, Unix.
+ """
+ return os.path.isdir(_makelongpath(path))
+
+
def islink(path):
- """Test whether a path is a symbolic link.
+ """os.path.islink(path) wrapper with support for long paths on Windows.
Availability: Windows, Unix.
"""
if isWindows():
import platform_utils_win32
- return platform_utils_win32.islink(path)
+ return platform_utils_win32.islink(_makelongpath(path))
else:
return os.path.islink(path)
@@ -288,7 +382,7 @@
"""
if isWindows():
import platform_utils_win32
- return platform_utils_win32.readlink(path)
+ return platform_utils_win32.readlink(_makelongpath(path))
else:
return os.readlink(path)