blob: b2cc24593f3dda2f5bff744844855ab09a853943 [file] [log] [blame]
Renaud Paquay2e702912016-11-01 11:23:38 -07001#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
Renaud Paquayad1abcb2016-11-01 11:34:55 -070016import errno
Renaud Paquay2e702912016-11-01 11:23:38 -070017import os
18import platform
19import select
Renaud Paquaya65adf72016-11-03 10:37:53 -070020import shutil
21import stat
Renaud Paquay2e702912016-11-01 11:23:38 -070022
Dylan Denge469a0c2018-06-23 15:02:26 +080023from pyversion import is_python3
24if is_python3():
25 from queue import Queue
26else:
27 from Queue import Queue
28
Renaud Paquay2e702912016-11-01 11:23:38 -070029from threading import Thread
30
31
32def isWindows():
33 """ Returns True when running with the native port of Python for Windows,
34 False when running on any other platform (including the Cygwin port of
35 Python).
36 """
37 # Note: The cygwin port of Python returns "CYGWIN_NT_xxx"
38 return platform.system() == "Windows"
39
40
41class FileDescriptorStreams(object):
42 """ Platform agnostic abstraction enabling non-blocking I/O over a
43 collection of file descriptors. This abstraction is required because
44 fctnl(os.O_NONBLOCK) is not supported on Windows.
45 """
46 @classmethod
47 def create(cls):
48 """ Factory method: instantiates the concrete class according to the
49 current platform.
50 """
51 if isWindows():
52 return _FileDescriptorStreamsThreads()
53 else:
54 return _FileDescriptorStreamsNonBlocking()
55
56 def __init__(self):
57 self.streams = []
58
59 def add(self, fd, dest, std_name):
60 """ Wraps an existing file descriptor as a stream.
61 """
62 self.streams.append(self._create_stream(fd, dest, std_name))
63
64 def remove(self, stream):
65 """ Removes a stream, when done with it.
66 """
67 self.streams.remove(stream)
68
69 @property
70 def is_done(self):
71 """ Returns True when all streams have been processed.
72 """
73 return len(self.streams) == 0
74
75 def select(self):
76 """ Returns the set of streams that have data available to read.
77 The returned streams each expose a read() and a close() method.
78 When done with a stream, call the remove(stream) method.
79 """
80 raise NotImplementedError
81
82 def _create_stream(fd, dest, std_name):
83 """ Creates a new stream wrapping an existing file descriptor.
84 """
85 raise NotImplementedError
86
87
88class _FileDescriptorStreamsNonBlocking(FileDescriptorStreams):
89 """ Implementation of FileDescriptorStreams for platforms that support
90 non blocking I/O.
91 """
92 class Stream(object):
93 """ Encapsulates a file descriptor """
94 def __init__(self, fd, dest, std_name):
95 self.fd = fd
96 self.dest = dest
97 self.std_name = std_name
98 self.set_non_blocking()
99
100 def set_non_blocking(self):
101 import fcntl
102 flags = fcntl.fcntl(self.fd, fcntl.F_GETFL)
103 fcntl.fcntl(self.fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
104
105 def fileno(self):
106 return self.fd.fileno()
107
108 def read(self):
109 return self.fd.read(4096)
110
111 def close(self):
112 self.fd.close()
113
114 def _create_stream(self, fd, dest, std_name):
115 return self.Stream(fd, dest, std_name)
116
117 def select(self):
118 ready_streams, _, _ = select.select(self.streams, [], [])
119 return ready_streams
120
121
122class _FileDescriptorStreamsThreads(FileDescriptorStreams):
123 """ Implementation of FileDescriptorStreams for platforms that don't support
124 non blocking I/O. This implementation requires creating threads issuing
125 blocking read operations on file descriptors.
126 """
127 def __init__(self):
128 super(_FileDescriptorStreamsThreads, self).__init__()
129 # The queue is shared accross all threads so we can simulate the
130 # behavior of the select() function
131 self.queue = Queue(10) # Limit incoming data from streams
132
133 def _create_stream(self, fd, dest, std_name):
134 return self.Stream(fd, dest, std_name, self.queue)
135
136 def select(self):
137 # Return only one stream at a time, as it is the most straighforward
138 # thing to do and it is compatible with the select() function.
139 item = self.queue.get()
140 stream = item.stream
141 stream.data = item.data
142 return [stream]
143
144 class QueueItem(object):
145 """ Item put in the shared queue """
146 def __init__(self, stream, data):
147 self.stream = stream
148 self.data = data
149
150 class Stream(object):
151 """ Encapsulates a file descriptor """
152 def __init__(self, fd, dest, std_name, queue):
153 self.fd = fd
154 self.dest = dest
155 self.std_name = std_name
156 self.queue = queue
157 self.data = None
158 self.thread = Thread(target=self.read_to_queue)
159 self.thread.daemon = True
160 self.thread.start()
161
162 def close(self):
163 self.fd.close()
164
165 def read(self):
166 data = self.data
167 self.data = None
168 return data
169
170 def read_to_queue(self):
171 """ The thread function: reads everything from the file descriptor into
172 the shared queue and terminates when reaching EOF.
173 """
174 for line in iter(self.fd.readline, b''):
175 self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, line))
176 self.fd.close()
177 self.queue.put(_FileDescriptorStreamsThreads.QueueItem(self, None))
Renaud Paquayd5cec5e2016-11-01 11:24:03 -0700178
179
180def symlink(source, link_name):
181 """Creates a symbolic link pointing to source named link_name.
182 Note: On Windows, source must exist on disk, as the implementation needs
183 to know whether to create a "File" or a "Directory" symbolic link.
184 """
185 if isWindows():
186 import platform_utils_win32
187 source = _validate_winpath(source)
188 link_name = _validate_winpath(link_name)
189 target = os.path.join(os.path.dirname(link_name), source)
Renaud Paquaybed8b622018-09-27 10:46:58 -0700190 if isdir(target):
191 platform_utils_win32.create_dirsymlink(_makelongpath(source), link_name)
Renaud Paquayd5cec5e2016-11-01 11:24:03 -0700192 else:
Renaud Paquaybed8b622018-09-27 10:46:58 -0700193 platform_utils_win32.create_filesymlink(_makelongpath(source), link_name)
Renaud Paquayd5cec5e2016-11-01 11:24:03 -0700194 else:
195 return os.symlink(source, link_name)
196
197
198def _validate_winpath(path):
199 path = os.path.normpath(path)
200 if _winpath_is_valid(path):
201 return path
202 raise ValueError("Path \"%s\" must be a relative path or an absolute "
203 "path starting with a drive letter".format(path))
204
205
206def _winpath_is_valid(path):
207 """Windows only: returns True if path is relative (e.g. ".\\foo") or is
208 absolute including a drive letter (e.g. "c:\\foo"). Returns False if path
209 is ambiguous (e.g. "x:foo" or "\\foo").
210 """
211 assert isWindows()
212 path = os.path.normpath(path)
213 drive, tail = os.path.splitdrive(path)
214 if tail:
215 if not drive:
216 return tail[0] != os.sep # "\\foo" is invalid
217 else:
218 return tail[0] == os.sep # "x:foo" is invalid
219 else:
220 return not drive # "x:" is invalid
Renaud Paquaya65adf72016-11-03 10:37:53 -0700221
222
Renaud Paquaybed8b622018-09-27 10:46:58 -0700223def _makelongpath(path):
224 """Return the input path normalized to support the Windows long path syntax
225 ("\\\\?\\" prefix) if needed, i.e. if the input path is longer than the
226 MAX_PATH limit.
227 """
Renaud Paquaya65adf72016-11-03 10:37:53 -0700228 if isWindows():
Renaud Paquaybed8b622018-09-27 10:46:58 -0700229 # Note: MAX_PATH is 260, but, for directories, the maximum value is actually 246.
230 if len(path) < 246:
231 return path
232 if path.startswith(u"\\\\?\\"):
233 return path
234 if not os.path.isabs(path):
235 return path
236 # Append prefix and ensure unicode so that the special longpath syntax
237 # is supported by underlying Win32 API calls
238 return u"\\\\?\\" + os.path.normpath(path)
239 else:
240 return path
241
242
243def rmtree(path):
244 """shutil.rmtree(path) wrapper with support for long paths on Windows.
245
246 Availability: Unix, Windows."""
247 if isWindows():
248 shutil.rmtree(_makelongpath(path), onerror=handle_rmtree_error)
Renaud Paquaya65adf72016-11-03 10:37:53 -0700249 else:
250 shutil.rmtree(path)
251
252
253def handle_rmtree_error(function, path, excinfo):
254 # Allow deleting read-only files
255 os.chmod(path, stat.S_IWRITE)
256 function(path)
Renaud Paquayad1abcb2016-11-01 11:34:55 -0700257
258
259def rename(src, dst):
Renaud Paquaybed8b622018-09-27 10:46:58 -0700260 """os.rename(src, dst) wrapper with support for long paths on Windows.
261
262 Availability: Unix, Windows."""
Renaud Paquayad1abcb2016-11-01 11:34:55 -0700263 if isWindows():
264 # On Windows, rename fails if destination exists, see
265 # https://docs.python.org/2/library/os.html#os.rename
266 try:
Renaud Paquaybed8b622018-09-27 10:46:58 -0700267 os.rename(_makelongpath(src), _makelongpath(dst))
Renaud Paquayad1abcb2016-11-01 11:34:55 -0700268 except OSError as e:
269 if e.errno == errno.EEXIST:
Renaud Paquaybed8b622018-09-27 10:46:58 -0700270 os.remove(_makelongpath(dst))
271 os.rename(_makelongpath(src), _makelongpath(dst))
Renaud Paquayad1abcb2016-11-01 11:34:55 -0700272 else:
273 raise
274 else:
275 os.rename(src, dst)
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700276
277
Renaud Paquay010fed72016-11-11 14:25:29 -0800278def remove(path):
Renaud Paquaybed8b622018-09-27 10:46:58 -0700279 """Remove (delete) the file path. This is a replacement for os.remove that
280 allows deleting read-only files on Windows, with support for long paths and
281 for deleting directory symbolic links.
282
283 Availability: Unix, Windows."""
Renaud Paquay010fed72016-11-11 14:25:29 -0800284 if isWindows():
Renaud Paquaybed8b622018-09-27 10:46:58 -0700285 longpath = _makelongpath(path)
Renaud Paquay010fed72016-11-11 14:25:29 -0800286 try:
Renaud Paquaybed8b622018-09-27 10:46:58 -0700287 os.remove(longpath)
Renaud Paquay010fed72016-11-11 14:25:29 -0800288 except OSError as e:
289 if e.errno == errno.EACCES:
Renaud Paquaybed8b622018-09-27 10:46:58 -0700290 os.chmod(longpath, stat.S_IWRITE)
291 # Directory symbolic links must be deleted with 'rmdir'.
292 if islink(longpath) and isdir(longpath):
293 os.rmdir(longpath)
294 else:
295 os.remove(longpath)
Renaud Paquay010fed72016-11-11 14:25:29 -0800296 else:
297 raise
298 else:
299 os.remove(path)
300
301
Renaud Paquaybed8b622018-09-27 10:46:58 -0700302def walk(top, topdown=True, onerror=None, followlinks=False):
303 """os.walk(path) wrapper with support for long paths on Windows.
304
305 Availability: Windows, Unix.
306 """
307 if isWindows():
308 return _walk_windows_impl(top, topdown, onerror, followlinks)
309 else:
310 return os.walk(top, topdown, onerror, followlinks)
311
312
313def _walk_windows_impl(top, topdown, onerror, followlinks):
314 try:
315 names = listdir(top)
316 except error, err:
317 if onerror is not None:
318 onerror(err)
319 return
320
321 dirs, nondirs = [], []
322 for name in names:
323 if isdir(os.path.join(top, name)):
324 dirs.append(name)
325 else:
326 nondirs.append(name)
327
328 if topdown:
329 yield top, dirs, nondirs
330 for name in dirs:
331 new_path = os.path.join(top, name)
332 if followlinks or not islink(new_path):
333 for x in _walk_windows_impl(new_path, topdown, onerror, followlinks):
334 yield x
335 if not topdown:
336 yield top, dirs, nondirs
337
338
339def listdir(path):
340 """os.listdir(path) wrapper with support for long paths on Windows.
341
342 Availability: Windows, Unix.
343 """
344 return os.listdir(_makelongpath(path))
345
346
347def rmdir(path):
348 """os.rmdir(path) wrapper with support for long paths on Windows.
349
350 Availability: Windows, Unix.
351 """
352 os.rmdir(_makelongpath(path))
353
354
355def isdir(path):
356 """os.path.isdir(path) wrapper with support for long paths on Windows.
357
358 Availability: Windows, Unix.
359 """
360 return os.path.isdir(_makelongpath(path))
361
362
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700363def islink(path):
Renaud Paquaybed8b622018-09-27 10:46:58 -0700364 """os.path.islink(path) wrapper with support for long paths on Windows.
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700365
366 Availability: Windows, Unix.
367 """
368 if isWindows():
369 import platform_utils_win32
Renaud Paquaybed8b622018-09-27 10:46:58 -0700370 return platform_utils_win32.islink(_makelongpath(path))
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700371 else:
372 return os.path.islink(path)
373
374
375def readlink(path):
376 """Return a string representing the path to which the symbolic link
377 points. The result may be either an absolute or relative pathname;
378 if it is relative, it may be converted to an absolute pathname using
379 os.path.join(os.path.dirname(path), result).
380
381 Availability: Windows, Unix.
382 """
383 if isWindows():
384 import platform_utils_win32
Renaud Paquaybed8b622018-09-27 10:46:58 -0700385 return platform_utils_win32.readlink(_makelongpath(path))
Renaud Paquay227ad2e2016-11-01 14:37:13 -0700386 else:
387 return os.readlink(path)
388
389
390def realpath(path):
391 """Return the canonical path of the specified filename, eliminating
392 any symbolic links encountered in the path.
393
394 Availability: Windows, Unix.
395 """
396 if isWindows():
397 current_path = os.path.abspath(path)
398 path_tail = []
399 for c in range(0, 100): # Avoid cycles
400 if islink(current_path):
401 target = readlink(current_path)
402 current_path = os.path.join(os.path.dirname(current_path), target)
403 else:
404 basename = os.path.basename(current_path)
405 if basename == '':
406 path_tail.append(current_path)
407 break
408 path_tail.append(basename)
409 current_path = os.path.dirname(current_path)
410 path_tail.reverse()
411 result = os.path.normpath(os.path.join(*path_tail))
412 return result
413 else:
414 return os.path.realpath(path)