adb-sync (31186B)
1 #!/usr/bin/env python3 2 3 # Copyright 2014 Google Inc. All rights reserved. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 """Sync files from/to an Android device.""" 17 18 from __future__ import unicode_literals 19 import argparse 20 import locale 21 import logging 22 import os 23 import re 24 import stat 25 import subprocess 26 import time 27 from types import TracebackType 28 from typing import Callable, cast, Dict, List, IO, Iterable, Optional, Tuple, Type 29 30 31 class OSLike(object): 32 33 def listdir(self, path: bytes) -> Iterable[bytes]: # os's name, so pylint: disable=g-bad-name 34 raise NotImplementedError('Abstract') 35 36 def lstat(self, path: bytes) -> os.stat_result: # os's name, so pylint: disable=g-bad-name 37 raise NotImplementedError('Abstract') 38 39 def stat(self, path: bytes) -> os.stat_result: # os's name, so pylint: disable=g-bad-name 40 raise NotImplementedError('Abstract') 41 42 def unlink(self, path: bytes) -> None: # os's name, so pylint: disable=g-bad-name 43 raise NotImplementedError('Abstract') 44 45 def rmdir(self, path: bytes) -> None: # os's name, so pylint: disable=g-bad-name 46 raise NotImplementedError('Abstract') 47 48 def makedirs(self, path: bytes) -> None: # os's name, so pylint: disable=g-bad-name 49 raise NotImplementedError('Abstract') 50 51 def utime(self, path: bytes, times: Tuple[float, float]) -> None: # os's name, so pylint: disable=g-bad-name 52 raise NotImplementedError('Abstract') 53 54 55 class GlobLike(object): 56 57 def glob(self, path: bytes) -> Iterable[bytes]: # glob's name, so pylint: disable=g-bad-name 58 raise NotImplementedError('Abstract') 59 60 61 class Stdout(object): 62 63 def __init__(self, args: List[bytes]) -> None: 64 """Closes the process's stdout when done. 65 66 Usage: 67 with Stdout(...) as stdout: 68 DoSomething(stdout) 69 70 Args: 71 args: Which program to run. 72 73 Returns: 74 An object for use by 'with'. 75 """ 76 self.popen = subprocess.Popen(args, stdout=subprocess.PIPE) 77 78 def __enter__(self) -> IO: 79 return self.popen.stdout 80 81 def __exit__(self, exc_type: Optional[Type[BaseException]], 82 exc_val: Optional[Exception], 83 exc_tb: Optional[TracebackType]) -> bool: 84 self.popen.stdout.close() 85 if self.popen.wait() != 0: 86 raise OSError('Subprocess exited with nonzero status.') 87 return False 88 89 90 class AdbFileSystem(GlobLike, OSLike): 91 """Mimics os's file interface but uses the adb utility.""" 92 93 def __init__(self, adb: List[bytes]) -> None: 94 self.stat_cache = {} # type: Dict[bytes, os.stat_result] 95 self.adb = adb 96 97 # Regarding parsing stat results, we only care for the following fields: 98 # - st_size 99 # - st_mtime 100 # - st_mode (but only about S_ISDIR and S_ISREG properties) 101 # Therefore, we only capture parts of 'ls -l' output that we actually use. 102 # The other fields will be filled with dummy values. 103 LS_TO_STAT_RE = re.compile( 104 br"""^ 105 (?: 106 (?P<S_IFREG> -) | 107 (?P<S_IFBLK> b) | 108 (?P<S_IFCHR> c) | 109 (?P<S_IFDIR> d) | 110 (?P<S_IFLNK> l) | 111 (?P<S_IFIFO> p) | 112 (?P<S_IFSOCK> s)) 113 [-r][-w][-xsS] 114 [-r][-w][-xsS] 115 [-r][-w][-xtT] # Mode string. 116 [ ]+ 117 (?: 118 [0-9]+ # number of hard links 119 [ ]+ 120 )? 121 [^ ]+ # User name/ID. 122 [ ]+ 123 [^ ]+ # Group name/ID. 124 [ ]+ 125 (?(S_IFBLK) [^ ]+[ ]+[^ ]+[ ]+) # Device numbers. 126 (?(S_IFCHR) [^ ]+[ ]+[^ ]+[ ]+) # Device numbers. 127 (?(S_IFDIR) [0-9]+ [ ]+)? # directory Size. 128 (?(S_IFREG) 129 (?P<st_size> [0-9]+) # Size. 130 [ ]+) 131 (?P<st_mtime> 132 [0-9]{4}-[0-9]{2}-[0-9]{2} # Date. 133 [ ] 134 [0-9]{2}:[0-9]{2}) # Time. 135 [ ] 136 # Don't capture filename for symlinks (ambiguous). 137 (?(S_IFLNK) .* | (?P<filename> .*)) 138 $""", re.DOTALL | re.VERBOSE) 139 140 def LsToStat(self, line: bytes) -> Tuple[os.stat_result, bytes]: 141 """Convert a line from 'ls -l' output to a stat result. 142 143 Args: 144 line: Output line of 'ls -l' on Android. 145 146 Returns: 147 os.stat_result for the line. 148 149 Raises: 150 OSError: if the given string is not a 'ls -l' output line (but maybe an 151 error message instead). 152 """ 153 154 match = self.LS_TO_STAT_RE.match(line) 155 if match is None: 156 logging.error('Could not parse %r.', line) 157 raise OSError('Unparseable ls -al result.') 158 groups = match.groupdict() 159 160 # Get the values we're interested in. 161 st_mode = ( # 0755 162 stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH 163 | stat.S_IXOTH) 164 if groups['S_IFREG']: 165 st_mode |= stat.S_IFREG 166 if groups['S_IFBLK']: 167 st_mode |= stat.S_IFBLK 168 if groups['S_IFCHR']: 169 st_mode |= stat.S_IFCHR 170 if groups['S_IFDIR']: 171 st_mode |= stat.S_IFDIR 172 if groups['S_IFIFO']: 173 st_mode |= stat.S_IFIFO 174 if groups['S_IFLNK']: 175 st_mode |= stat.S_IFLNK 176 if groups['S_IFSOCK']: 177 st_mode |= stat.S_IFSOCK 178 st_size = None if groups['st_size'] is None else int(groups['st_size']) 179 st_mtime = int( 180 time.mktime( 181 time.strptime( 182 match.group('st_mtime').decode('ascii'), '%Y-%m-%d %H:%M'))) 183 184 # Fill the rest with dummy values. 185 st_ino = 1 186 st_rdev = 0 187 st_nlink = 1 188 st_uid = -2 # Nobody. 189 st_gid = -2 # Nobody. 190 st_atime = st_ctime = st_mtime 191 192 stbuf = os.stat_result((st_mode, st_ino, st_rdev, st_nlink, st_uid, st_gid, 193 st_size, st_atime, st_mtime, st_ctime)) 194 filename = groups['filename'] 195 return stbuf, filename 196 197 def QuoteArgument(self, arg: bytes) -> bytes: 198 # Quotes an argument for use by adb shell. 199 # Usually, arguments in 'adb shell' use are put in double quotes by adb, 200 # but not in any way escaped. 201 arg = arg.replace(b'\\', b'\\\\') 202 arg = arg.replace(b'"', b'\\"') 203 arg = arg.replace(b'$', b'\\$') 204 arg = arg.replace(b'`', b'\\`') 205 arg = b'"' + arg + b'"' 206 return arg 207 208 def IsWorking(self) -> bool: 209 """Tests the adb connection.""" 210 # This string should contain all possible evil, but no percent signs. 211 # Note this code uses 'date' and not 'echo', as date just calls strftime 212 # while echo does its own backslash escape handling additionally to the 213 # shell's. Too bad printf "%s\n" is not available. 214 test_strings = [ 215 b'(', b'(; #`ls`$PATH\'"(\\\\\\\\){};!\xc0\xaf\xff\xc2\xbf' 216 ] 217 for test_string in test_strings: 218 good = False 219 with Stdout(self.adb + 220 [b'shell', 221 b'date +%s' % (self.QuoteArgument(test_string),)]) as stdout: 222 for line in stdout: 223 line = line.rstrip(b'\r\n') 224 if line == test_string: 225 good = True 226 if not good: 227 return False 228 return True 229 230 def listdir(self, path: bytes) -> Iterable[bytes]: # os's name, so pylint: disable=g-bad-name 231 """List the contents of a directory, caching them for later lstat calls.""" 232 with Stdout(self.adb + 233 [b'shell', 234 b'ls -al %s' % (self.QuoteArgument(path + b'/'),)]) as stdout: 235 for line in stdout: 236 if line.startswith(b'total '): 237 continue 238 line = line.rstrip(b'\r\n') 239 try: 240 statdata, filename = self.LsToStat(line) 241 except OSError: 242 continue 243 if filename is None: 244 logging.error('Could not parse %r.', line) 245 else: 246 self.stat_cache[path + b'/' + filename] = statdata 247 yield filename 248 249 def lstat(self, path: bytes) -> os.stat_result: # os's name, so pylint: disable=g-bad-name 250 """Stat a file.""" 251 if path in self.stat_cache: 252 return self.stat_cache[path] 253 with Stdout( 254 self.adb + 255 [b'shell', b'ls -ald %s' % (self.QuoteArgument(path),)]) as stdout: 256 for line in stdout: 257 if line.startswith(b'total '): 258 continue 259 line = line.rstrip(b'\r\n') 260 statdata, _ = self.LsToStat(line) 261 self.stat_cache[path] = statdata 262 return statdata 263 raise OSError('No such file or directory') 264 265 def stat(self, path: bytes) -> os.stat_result: # os's name, so pylint: disable=g-bad-name 266 """Stat a file.""" 267 if path in self.stat_cache and not stat.S_ISLNK( 268 self.stat_cache[path].st_mode): 269 return self.stat_cache[path] 270 with Stdout( 271 self.adb + 272 [b'shell', b'ls -aldL %s' % (self.QuoteArgument(path),)]) as stdout: 273 for line in stdout: 274 if line.startswith(b'total '): 275 continue 276 line = line.rstrip(b'\r\n') 277 statdata, _ = self.LsToStat(line) 278 self.stat_cache[path] = statdata 279 return statdata 280 raise OSError('No such file or directory') 281 282 def unlink(self, path: bytes) -> None: # os's name, so pylint: disable=g-bad-name 283 """Delete a file.""" 284 if subprocess.call( 285 self.adb + [b'shell', b'rm %s' % (self.QuoteArgument(path),)]) != 0: 286 raise OSError('unlink failed') 287 288 def rmdir(self, path: bytes) -> None: # os's name, so pylint: disable=g-bad-name 289 """Delete a directory.""" 290 if subprocess.call( 291 self.adb + 292 [b'shell', b'rmdir %s' % (self.QuoteArgument(path),)]) != 0: 293 raise OSError('rmdir failed') 294 295 def makedirs(self, path: bytes) -> None: # os's name, so pylint: disable=g-bad-name 296 """Create a directory.""" 297 if subprocess.call( 298 self.adb + 299 [b'shell', b'mkdir -p %s' % (self.QuoteArgument(path),)]) != 0: 300 raise OSError('mkdir failed') 301 302 def utime(self, path: bytes, times: Tuple[float, float]) -> None: 303 # TODO(rpolzer): Find out why this does not work (returns status 255). 304 """Set the time of a file to a specified unix time.""" 305 atime, mtime = times 306 timestr = time.strftime('%Y%m%d.%H%M%S', 307 time.localtime(mtime)).encode('ascii') 308 if subprocess.call( 309 self.adb + 310 [b'shell', 311 b'touch -mt %s %s' % (timestr, self.QuoteArgument(path))]) != 0: 312 raise OSError('touch failed') 313 timestr = time.strftime('%Y%m%d.%H%M%S', 314 time.localtime(atime)).encode('ascii') 315 if subprocess.call( 316 self.adb + 317 [b'shell', 318 b'touch -at %s %s' % (timestr, self.QuoteArgument(path))]) != 0: 319 raise OSError('touch failed') 320 321 def glob(self, path: bytes) -> Iterable[bytes]: # glob's name, so pylint: disable=g-bad-name 322 with Stdout( 323 self.adb + 324 [b'shell', b'for p in %s; do echo "$p"; done' % (path,)]) as stdout: 325 for line in stdout: 326 yield line.rstrip(b'\r\n') 327 328 def Push(self, src: bytes, dst: bytes) -> None: 329 """Push a file from the local file system to the Android device.""" 330 if subprocess.call(self.adb + [b'push', src, dst]) != 0: 331 raise OSError('push failed') 332 333 def Pull(self, src: bytes, dst: bytes) -> None: 334 """Pull a file from the Android device to the local file system.""" 335 if subprocess.call(self.adb + [b'pull', src, dst]) != 0: 336 raise OSError('pull failed') 337 338 339 def BuildFileList(fs: OSLike, path: bytes, follow_links: bool, 340 prefix: bytes) -> Iterable[Tuple[bytes, os.stat_result]]: 341 """Builds a file list. 342 343 Args: 344 fs: File system provider (can be os or AdbFileSystem()). 345 path: Initial path. 346 follow_links: Whether to follow symlinks while iterating. May recurse 347 endlessly. 348 prefix: Path prefix for output file names. 349 350 Yields: 351 File names from path (prefixed by prefix). 352 Directories are yielded before their contents. 353 """ 354 try: 355 if follow_links: 356 statresult = fs.stat(path) 357 else: 358 statresult = fs.lstat(path) 359 except OSError: 360 return 361 if stat.S_ISDIR(statresult.st_mode): 362 yield prefix, statresult 363 try: 364 files = fs.listdir(path) 365 except OSError: 366 return 367 for n in files: 368 if n == b'.' or n == b'..': 369 continue 370 for t in BuildFileList(fs, path + b'/' + n, follow_links, 371 prefix + b'/' + n): 372 yield t 373 elif stat.S_ISREG(statresult.st_mode): 374 yield prefix, statresult 375 elif stat.S_ISLNK(statresult.st_mode) and not follow_links: 376 yield prefix, statresult 377 else: 378 logging.info('Unsupported file: %r.', path) 379 380 381 def DiffLists(a: Iterable[Tuple[bytes, os.stat_result]], 382 b: Iterable[Tuple[bytes, os.stat_result]] 383 ) -> Tuple[List[Tuple[bytes, os.stat_result]], List[ 384 Tuple[bytes, os.stat_result, os 385 .stat_result]], List[Tuple[bytes, os.stat_result]]]: 386 """Compares two lists. 387 388 Args: 389 a: the first list. 390 b: the second list. 391 392 Returns: 393 a_only: the items from list a. 394 both: the items from both list, with the remaining tuple items combined. 395 b_only: the items from list b. 396 """ 397 a_only = [] # type: List[Tuple[bytes, os.stat_result]] 398 b_only = [] # type: List[Tuple[bytes, os.stat_result]] 399 both = [] # type: List[Tuple[bytes, os.stat_result, os.stat_result]] 400 401 a_revlist = sorted(a) 402 a_revlist.reverse() 403 b_revlist = sorted(b) 404 b_revlist.reverse() 405 406 while True: 407 if not a_revlist: 408 b_only.extend(reversed(b_revlist)) 409 break 410 if not b_revlist: 411 a_only.extend(reversed(a_revlist)) 412 break 413 a_item = a_revlist[len(a_revlist) - 1] 414 b_item = b_revlist[len(b_revlist) - 1] 415 if a_item[0] == b_item[0]: 416 both.append((a_item[0], a_item[1], b_item[1])) 417 a_revlist.pop() 418 b_revlist.pop() 419 elif a_item[0] < b_item[0]: 420 a_only.append(a_item) 421 a_revlist.pop() 422 elif a_item[0] > b_item[0]: 423 b_only.append(b_item) 424 b_revlist.pop() 425 else: 426 raise 427 428 return a_only, both, b_only 429 430 431 class DeleteInterruptedFile(object): 432 433 def __init__(self, dry_run: bool, fs: OSLike, name: bytes) -> None: 434 """Sets up interrupt protection. 435 436 Usage: 437 with DeleteInterruptedFile(False, fs, name): 438 DoSomething() 439 440 If DoSomething() should get interrupted, the file 'name' will be deleted. 441 The exception otherwise will be passed on. 442 443 Args: 444 dry_run: If true, we don't actually delete. 445 fs: File system object. 446 name: File name to delete. 447 448 Returns: 449 An object for use by 'with'. 450 """ 451 self.dry_run = dry_run 452 self.fs = fs 453 self.name = name 454 455 def __enter__(self) -> None: 456 pass 457 458 def __exit__(self, exc_type: Optional[Type[BaseException]], 459 exc_val: Optional[Exception], 460 exc_tb: Optional[TracebackType]) -> bool: 461 if exc_type is not None: 462 logging.info('Interrupted-%s-Delete: %r', 463 'Pull' if self.fs == os else 'Push', self.name) 464 if not self.dry_run: 465 self.fs.unlink(self.name) 466 return False 467 468 469 class FileSyncer(object): 470 """File synchronizer.""" 471 472 def __init__(self, adb: AdbFileSystem, local_path: bytes, remote_path: bytes, 473 local_to_remote: bool, remote_to_local: bool, 474 preserve_times: bool, delete_missing: bool, 475 allow_overwrite: bool, allow_replace: bool, copy_links: bool, 476 dry_run: bool) -> None: 477 self.local = local_path 478 self.remote = remote_path 479 self.adb = adb 480 self.local_to_remote = local_to_remote 481 self.remote_to_local = remote_to_local 482 self.preserve_times = preserve_times 483 self.delete_missing = delete_missing 484 self.allow_overwrite = allow_overwrite 485 self.allow_replace = allow_replace 486 self.copy_links = copy_links 487 self.dry_run = dry_run 488 self.num_bytes = 0 489 self.start_time = time.time() 490 491 # Attributes filled in later. 492 local_only = None # type: List[Tuple[bytes, os.stat_result]] 493 both = None # type: List[Tuple[bytes, os.stat_result, os.stat_result]] 494 remote_only = None # type: List[Tuple[bytes, os.stat_result]] 495 src_to_dst = None # type: Tuple[bool, bool] 496 dst_to_src = None # type: Tuple[bool, bool] 497 src_only = None # type: Tuple[List[Tuple[bytes, os.stat_result]], List[Tuple[bytes, os.stat_result]]] 498 dst_only = None # type: Tuple[List[Tuple[bytes, os.stat_result]], List[Tuple[bytes, os.stat_result]]] 499 src = None # type: Tuple[bytes, bytes] 500 dst = None # type: Tuple[bytes, bytes] 501 dst_fs = None # type: Tuple[OSLike, OSLike] 502 push = None # type: Tuple[str, str] 503 copy = None # type: Tuple[Callable[[bytes, bytes], None], Callable[[bytes, bytes], None]] 504 505 def IsWorking(self) -> bool: 506 """Tests the adb connection.""" 507 return self.adb.IsWorking() 508 509 def ScanAndDiff(self) -> None: 510 """Scans the local and remote locations and identifies differences.""" 511 logging.info('Scanning and diffing...') 512 locallist = BuildFileList( 513 cast(OSLike, os), self.local, self.copy_links, b'') 514 remotelist = BuildFileList(self.adb, self.remote, self.copy_links, b'') 515 self.local_only, self.both, self.remote_only = DiffLists( 516 locallist, remotelist) 517 if not self.local_only and not self.both and not self.remote_only: 518 logging.warning('No files seen. User error?') 519 self.src_to_dst = (self.local_to_remote, self.remote_to_local) 520 self.dst_to_src = (self.remote_to_local, self.local_to_remote) 521 self.src_only = (self.local_only, self.remote_only) 522 self.dst_only = (self.remote_only, self.local_only) 523 self.src = (self.local, self.remote) 524 self.dst = (self.remote, self.local) 525 self.dst_fs = (self.adb, cast(OSLike, os)) 526 self.push = ('Push', 'Pull') 527 self.copy = (self.adb.Push, self.adb.Pull) 528 529 def PerformDeletions(self) -> None: 530 """Perform all deleting necessary for the file sync operation.""" 531 if not self.delete_missing: 532 return 533 for i in [0, 1]: 534 if self.src_to_dst[i] and not self.dst_to_src[i]: 535 if not self.src_only[i] and not self.both: 536 logging.error('Cowardly refusing to delete everything.') 537 else: 538 for name, s in reversed(self.dst_only[i]): 539 dst_name = self.dst[i] + name 540 logging.info('%s-Delete: %r', self.push[i], dst_name) 541 if stat.S_ISDIR(s.st_mode): 542 if not self.dry_run: 543 self.dst_fs[i].rmdir(dst_name) 544 else: 545 if not self.dry_run: 546 self.dst_fs[i].unlink(dst_name) 547 del self.dst_only[i][:] 548 549 def PerformOverwrites(self) -> None: 550 """Delete files/directories that are in the way for overwriting.""" 551 src_only_prepend = ( 552 [], [] 553 ) # type: Tuple[List[Tuple[bytes, os.stat_result]], List[Tuple[bytes, os.stat_result]]] 554 for name, localstat, remotestat in self.both: 555 if stat.S_ISDIR(localstat.st_mode) and stat.S_ISDIR(remotestat.st_mode): 556 # A dir is a dir is a dir. 557 continue 558 elif stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): 559 # Dir vs file? Nothing to do here yet. 560 pass 561 else: 562 # File vs file? Compare sizes. 563 if localstat.st_size == remotestat.st_size: 564 continue 565 l2r = self.local_to_remote 566 r2l = self.remote_to_local 567 if l2r and r2l: 568 # Truncate times to full minutes, as Android's "ls" only outputs minute 569 # accuracy. 570 localminute = int(localstat.st_mtime / 60) 571 remoteminute = int(remotestat.st_mtime / 60) 572 if localminute > remoteminute: 573 r2l = False 574 elif localminute < remoteminute: 575 l2r = False 576 if l2r and r2l: 577 logging.warning('Unresolvable: %r', name) 578 continue 579 if l2r: 580 i = 0 # Local to remote operation. 581 src_stat = localstat 582 dst_stat = remotestat 583 else: 584 i = 1 # Remote to local operation. 585 src_stat = remotestat 586 dst_stat = localstat 587 dst_name = self.dst[i] + name 588 logging.info('%s-Delete-Conflicting: %r', self.push[i], dst_name) 589 if stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): 590 if not self.allow_replace: 591 logging.info('Would have to replace to do this. ' 592 'Use --force to allow this.') 593 continue 594 if not self.allow_overwrite: 595 logging.info('Would have to overwrite to do this, ' 596 'which --no-clobber forbids.') 597 continue 598 if stat.S_ISDIR(dst_stat.st_mode): 599 kill_files = [ 600 x for x in self.dst_only[i] if x[0][:len(name) + 1] == name + b'/' 601 ] 602 self.dst_only[i][:] = [ 603 x for x in self.dst_only[i] if x[0][:len(name) + 1] != name + b'/' 604 ] 605 for l, s in reversed(kill_files): 606 if stat.S_ISDIR(s.st_mode): 607 if not self.dry_run: 608 self.dst_fs[i].rmdir(self.dst[i] + l) 609 else: 610 if not self.dry_run: 611 self.dst_fs[i].unlink(self.dst[i] + l) 612 if not self.dry_run: 613 self.dst_fs[i].rmdir(dst_name) 614 elif stat.S_ISDIR(src_stat.st_mode): 615 if not self.dry_run: 616 self.dst_fs[i].unlink(dst_name) 617 else: 618 if not self.dry_run: 619 self.dst_fs[i].unlink(dst_name) 620 src_only_prepend[i].append((name, src_stat)) 621 for i in [0, 1]: 622 self.src_only[i][:0] = src_only_prepend[i] 623 624 def PerformCopies(self) -> None: 625 """Perform all copying necessary for the file sync operation.""" 626 for i in [0, 1]: 627 if self.src_to_dst[i]: 628 for name, s in self.src_only[i]: 629 src_name = self.src[i] + name 630 dst_name = self.dst[i] + name 631 logging.info('%s: %r', self.push[i], dst_name) 632 if stat.S_ISDIR(s.st_mode): 633 if not self.dry_run: 634 self.dst_fs[i].makedirs(dst_name) 635 else: 636 with DeleteInterruptedFile(self.dry_run, self.dst_fs[i], dst_name): 637 if not self.dry_run: 638 self.copy[i](src_name, dst_name) 639 if stat.S_ISREG(s.st_mode): 640 self.num_bytes += s.st_size 641 if not self.dry_run: 642 if self.preserve_times: 643 logging.info('%s-Times: accessed %s, modified %s', self.push[i], 644 time.asctime(time.localtime(s.st_atime)), 645 time.asctime(time.localtime(s.st_mtime))) 646 self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime)) 647 648 def TimeReport(self) -> None: 649 """Report time and amount of data transferred.""" 650 if self.dry_run: 651 logging.info('Total: %d bytes', self.num_bytes) 652 else: 653 end_time = time.time() 654 dt = end_time - self.start_time 655 rate = self.num_bytes / 1024.0 / dt 656 logging.info('Total: %d KB/s (%d bytes in %.3fs)', rate, self.num_bytes, 657 dt) 658 659 660 def ExpandWildcards(globber: GlobLike, path: bytes) -> Iterable[bytes]: 661 if path.find(b'?') == -1 and path.find(b'*') == -1 and path.find(b'[') == -1: 662 return [path] 663 return globber.glob(path) 664 665 666 def FixPath(src: bytes, dst: bytes) -> Tuple[bytes, bytes]: 667 # rsync-like path munging to make remote specifications shorter. 668 append = b'' 669 pos = src.rfind(b'/') 670 if pos >= 0: 671 if src.endswith(b'/'): 672 # Final slash: copy to the destination "as is". 673 pass 674 else: 675 # No final slash: destination name == source name. 676 append = src[pos:] 677 else: 678 # No slash at all - use same name at destination. 679 append = b'/' + src 680 # Append the destination file name if any. 681 # BUT: do not append "." or ".." components! 682 if append != b'/.' and append != b'/..': 683 dst += append 684 return (src, dst) 685 686 687 def main() -> None: 688 logging.basicConfig(level=logging.INFO) 689 690 parser = argparse.ArgumentParser( 691 description='Synchronize a directory between an Android device and the ' 692 'local file system') 693 parser.add_argument( 694 'source', 695 metavar='SRC', 696 type=str, 697 nargs='+', 698 help='The directory to read files/directories from. ' 699 'This must be a local path if -R is not specified, ' 700 'and an Android path if -R is specified. If SRC does ' 701 'not end with a final slash, its last path component ' 702 'is appended to DST (like rsync does).') 703 parser.add_argument( 704 'destination', 705 metavar='DST', 706 type=str, 707 help='The directory to write files/directories to. ' 708 'This must be an Android path if -R is not specified, ' 709 'and a local path if -R is specified.') 710 parser.add_argument( 711 '-e', 712 '--adb', 713 metavar='COMMAND', 714 default='adb', 715 type=str, 716 help='Use the given adb binary and arguments.') 717 parser.add_argument( 718 '--device', 719 action='store_true', 720 help='Directs command to the only connected USB device; ' 721 'returns an error if more than one USB device is present. ' 722 'Corresponds to the "-d" option of adb.') 723 parser.add_argument( 724 '--emulator', 725 action='store_true', 726 help='Directs command to the only running emulator; ' 727 'returns an error if more than one emulator is running. ' 728 'Corresponds to the "-e" option of adb.') 729 parser.add_argument( 730 '-s', 731 '--serial', 732 metavar='DEVICE', 733 type=str, 734 help='Directs command to the device or emulator with ' 735 'the given serial number or qualifier. Overrides ' 736 'ANDROID_SERIAL environment variable. Use "adb devices" ' 737 'to list all connected devices with their respective serial number. ' 738 'Corresponds to the "-s" option of adb.') 739 parser.add_argument( 740 '-H', 741 '--host', 742 metavar='HOST', 743 type=str, 744 help='Name of adb server host (default: localhost). ' 745 'Corresponds to the "-H" option of adb.') 746 parser.add_argument( 747 '-P', 748 '--port', 749 metavar='PORT', 750 type=str, 751 help='Port of adb server (default: 5037). ' 752 'Corresponds to the "-P" option of adb.') 753 parser.add_argument( 754 '-R', 755 '--reverse', 756 action='store_true', 757 help='Reverse sync (pull, not push).') 758 parser.add_argument( 759 '-2', 760 '--two-way', 761 action='store_true', 762 help='Two-way sync (compare modification time; after ' 763 'the sync, both sides will have all files in the ' 764 'respective newest version. This relies on the clocks ' 765 'of your system and the device to match.') 766 parser.add_argument( 767 '-t', 768 '--times', 769 action='store_true', 770 help='Preserve modification times when copying.') 771 parser.add_argument( 772 '-d', 773 '--delete', 774 action='store_true', 775 help='Delete files from DST that are not present on ' 776 'SRC. Mutually exclusive with -2.') 777 parser.add_argument( 778 '-f', 779 '--force', 780 action='store_true', 781 help='Allow deleting files/directories when having to ' 782 'replace a file by a directory or vice versa. This is ' 783 'disabled by default to prevent large scale accidents.') 784 parser.add_argument( 785 '-n', 786 '--no-clobber', 787 action='store_true', 788 help='Do not ever overwrite any ' 789 'existing files. Mutually exclusive with -f.') 790 parser.add_argument( 791 '-L', 792 '--copy-links', 793 action='store_true', 794 help='transform symlink into referent file/dir') 795 parser.add_argument( 796 '--dry-run', 797 action='store_true', 798 help='Do not do anything - just show what would be done.') 799 args = parser.parse_args() 800 801 localpatterns = [os.fsencode(x) for x in args.source] 802 remotepath = os.fsencode(args.destination) 803 adb_args = os.fsencode(args.adb).split(b' ') 804 if args.device: 805 adb_args += [b'-d'] 806 if args.emulator: 807 adb_args += [b'-e'] 808 if args.serial: 809 adb_args += [b'-s', os.fsencode(args.serial)] 810 if args.host: 811 adb_args += [b'-H', os.fsencode(args.host)] 812 if args.port: 813 adb_args += [b'-P', os.fsencode(args.port)] 814 adb = AdbFileSystem(adb_args) 815 816 # Expand wildcards, but only on the remote side. 817 localpaths = [] 818 remotepaths = [] 819 if args.reverse: 820 for pattern in localpatterns: 821 for src in ExpandWildcards(adb, pattern): 822 src, dst = FixPath(src, remotepath) 823 localpaths.append(src) 824 remotepaths.append(dst) 825 else: 826 for src in localpatterns: 827 src, dst = FixPath(src, remotepath) 828 localpaths.append(src) 829 remotepaths.append(dst) 830 831 preserve_times = args.times 832 delete_missing = args.delete 833 allow_replace = args.force 834 allow_overwrite = not args.no_clobber 835 copy_links = args.copy_links 836 dry_run = args.dry_run 837 local_to_remote = True 838 remote_to_local = False 839 if args.two_way: 840 local_to_remote = True 841 remote_to_local = True 842 if args.reverse: 843 local_to_remote, remote_to_local = remote_to_local, local_to_remote 844 localpaths, remotepaths = remotepaths, localpaths 845 if allow_replace and not allow_overwrite: 846 logging.error('--no-clobber and --force are mutually exclusive.') 847 parser.print_help() 848 return 849 if delete_missing and local_to_remote and remote_to_local: 850 logging.error('--delete and --two-way are mutually exclusive.') 851 parser.print_help() 852 return 853 854 # Two-way sync is only allowed with disjoint remote and local path sets. 855 if (remote_to_local and local_to_remote) or delete_missing: 856 if ((remote_to_local and len(localpaths) != len(set(localpaths))) or 857 (local_to_remote and len(remotepaths) != len(set(remotepaths)))): 858 logging.error( 859 '--two-way and --delete are only supported for disjoint sets of ' 860 'source and destination paths (in other words, all SRC must ' 861 'differ in basename).') 862 parser.print_help() 863 return 864 865 for i in range(len(localpaths)): 866 logging.info('Sync: local %r, remote %r', localpaths[i], remotepaths[i]) 867 syncer = FileSyncer(adb, localpaths[i], remotepaths[i], local_to_remote, 868 remote_to_local, preserve_times, delete_missing, 869 allow_overwrite, allow_replace, copy_links, dry_run) 870 if not syncer.IsWorking(): 871 logging.error('Device not connected or not working.') 872 return 873 try: 874 syncer.ScanAndDiff() 875 syncer.PerformDeletions() 876 syncer.PerformOverwrites() 877 syncer.PerformCopies() 878 finally: 879 syncer.TimeReport() 880 881 882 if __name__ == '__main__': 883 main()