init

.pydevproject

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/git-repo</path>
+</pydev_pathproperty>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+</pydev_project>

README.md

+# repo
+
+Repo is a tool built on top of Git.  Repo helps manage many Git repositories,
+does the uploads to revision control systems, and automates parts of the
+development workflow.  Repo is not meant to replace Git, only to make it
+easier to work with Git.  The repo command is an executable Python script
+that you can put anywhere in your path.
+
+* Homepage: https://gerrit.googlesource.com/git-repo/
+* Bug reports: https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo
+* Source: https://gerrit.googlesource.com/git-repo/
+* Overview: https://source.android.com/source/developing.html
+* Docs: https://source.android.com/source/using-repo.html
+* [repo Manifest Format](./docs/manifest-format.md)
+* [repo Hooks](./docs/repo-hooks.md)
+* [Submitting patches](./SUBMITTING_PATCHES.md)

color.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+import pager
+
+COLORS = {None: -1,
+          'normal': -1,
+          'black': 0,
+          'red': 1,
+          'green': 2,
+          'yellow': 3,
+          'blue': 4,
+          'magenta': 5,
+          'cyan': 6,
+          'white': 7}
+
+ATTRS = {None: -1,
+         'bold': 1,
+         'dim': 2,
+         'ul': 4,
+         'blink': 5,
+         'reverse': 7}
+
+RESET = "\033[m"
+
+
+def is_color(s):
+  return s in COLORS
+
+
+def is_attr(s):
+  return s in ATTRS
+
+
+def _Color(fg=None, bg=None, attr=None):
+  fg = COLORS[fg]
+  bg = COLORS[bg]
+  attr = ATTRS[attr]
+
+  if attr >= 0 or fg >= 0 or bg >= 0:
+    need_sep = False
+    code = "\033["
+
+    if attr >= 0:
+      code += chr(ord('0') + attr)
+      need_sep = True
+
+    if fg >= 0:
+      if need_sep:
+        code += ';'
+      need_sep = True
+
+      if fg < 8:
+        code += '3%c' % (ord('0') + fg)
+      else:
+        code += '38;5;%d' % fg
+
+    if bg >= 0:
+      if need_sep:
+        code += ';'
+
+      if bg < 8:
+        code += '4%c' % (ord('0') + bg)
+      else:
+        code += '48;5;%d' % bg
+    code += 'm'
+  else:
+    code = ''
+  return code
+
+DEFAULT = None
+
+
+def SetDefaultColoring(state):
+  """Set coloring behavior to |state|.
+
+  This is useful for overriding config options via the command line.
+  """
+  if state is None:
+    # Leave it alone -- return quick!
+    return
+
+  global DEFAULT
+  state = state.lower()
+  if state in ('auto',):
+    DEFAULT = state
+  elif state in ('always', 'yes', 'true', True):
+    DEFAULT = 'always'
+  elif state in ('never', 'no', 'false', False):
+    DEFAULT = 'never'
+
+
+class Coloring(object):
+  def __init__(self, config, section_type):
+    self._section = 'color.%s' % section_type
+    self._config = config
+    self._out = sys.stdout
+
+    on = DEFAULT
+    if on is None:
+      on = self._config.GetString(self._section)
+      if on is None:
+        on = self._config.GetString('color.ui')
+
+    if on == 'auto':
+      if pager.active or os.isatty(1):
+        self._on = True
+      else:
+        self._on = False
+    elif on in ('true', 'always'):
+      self._on = True
+    else:
+      self._on = False
+
+  def redirect(self, out):
+    self._out = out
+
+  @property
+  def is_on(self):
+    return self._on
+
+  def write(self, fmt, *args):
+    self._out.write(fmt % args)
+
+  def flush(self):
+    self._out.flush()
+
+  def nl(self):
+    self._out.write('\n')
+
+  def printer(self, opt=None, fg=None, bg=None, attr=None):
+    s = self
+    c = self.colorer(opt, fg, bg, attr)
+
+    def f(fmt, *args):
+      s._out.write(c(fmt, *args))
+    return f
+
+  def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
+    s = self
+    c = self.nofmt_colorer(opt, fg, bg, attr)
+
+    def f(fmt):
+      s._out.write(c(fmt))
+    return f
+
+  def colorer(self, opt=None, fg=None, bg=None, attr=None):
+    if self._on:
+      c = self._parse(opt, fg, bg, attr)
+
+      def f(fmt, *args):
+        output = fmt % args
+        return ''.join([c, output, RESET])
+      return f
+    else:
+
+      def f(fmt, *args):
+        return fmt % args
+      return f
+
+  def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
+    if self._on:
+      c = self._parse(opt, fg, bg, attr)
+
+      def f(fmt):
+        return ''.join([c, fmt, RESET])
+      return f
+    else:
+      def f(fmt):
+        return fmt
+      return f
+
+  def _parse(self, opt, fg, bg, attr):
+    if not opt:
+      return _Color(fg, bg, attr)
+
+    v = self._config.GetString('%s.%s' % (self._section, opt))
+    if v is None:
+      return _Color(fg, bg, attr)
+
+    v = v.strip().lower()
+    if v == "reset":
+      return RESET
+    elif v == '':
+      return _Color(fg, bg, attr)
+
+    have_fg = False
+    for a in v.split(' '):
+      if is_color(a):
+        if have_fg:
+          bg = a
+        else:
+          fg = a
+      elif is_attr(a):
+        attr = a
+
+    return _Color(fg, bg, attr)

command.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import optparse
+import platform
+import re
+import sys
+
+from event_log import EventLog
+from error import NoSuchProjectError
+from error import InvalidProjectGroupsError
+
+
+class Command(object):
+  """Base class for any command line action in repo.
+  """
+
+  common = False
+  event_log = EventLog()
+  manifest = None
+  _optparse = None
+
+  def WantPager(self, _opt):
+    return False
+
+  def ReadEnvironmentOptions(self, opts):
+    """ Set options from environment variables. """
+
+    env_options = self._RegisteredEnvironmentOptions()
+
+    for env_key, opt_key in env_options.items():
+      # Get the user-set option value if any
+      opt_value = getattr(opts, opt_key)
+
+      # If the value is set, it means the user has passed it as a command
+      # line option, and we should use that.  Otherwise we can try to set it
+      # with the value from the corresponding environment variable.
+      if opt_value is not None:
+        continue
+
+      env_value = os.environ.get(env_key)
+      if env_value is not None:
+        setattr(opts, opt_key, env_value)
+
+    return opts
+
+  @property
+  def OptionParser(self):
+    if self._optparse is None:
+      try:
+        me = 'repo %s' % self.NAME
+        usage = self.helpUsage.strip().replace('%prog', me)
+      except AttributeError:
+        usage = 'repo %s' % self.NAME
+      self._optparse = optparse.OptionParser(usage=usage)
+      self._Options(self._optparse)
+    return self._optparse
+
+  def _Options(self, p):
+    """Initialize the option parser.
+    """
+
+  def _RegisteredEnvironmentOptions(self):
+    """Get options that can be set from environment variables.
+
+    Return a dictionary mapping environment variable name
+    to option key name that it can override.
+
+    Example: {'REPO_MY_OPTION': 'my_option'}
+
+    Will allow the option with key value 'my_option' to be set
+    from the value in the environment variable named 'REPO_MY_OPTION'.
+
+    Note: This does not work properly for options that are explicitly
+    set to None by the user, or options that are defined with a
+    default value other than None.
+
+    """
+    return {}
+
+  def Usage(self):
+    """Display usage and terminate.
+    """
+    self.OptionParser.print_usage()
+    sys.exit(1)
+
+  def ValidateOptions(self, opt, args):
+    """Validate the user options & arguments before executing.
+
+    This is meant to help break the code up into logical steps.  Some tips:
+    * Use self.OptionParser.error to display CLI related errors.
+    * Adjust opt member defaults as makes sense.
+    * Adjust the args list, but do so inplace so the caller sees updates.
+    * Try to avoid updating self state.  Leave that to Execute.
+    """
+
+  def Execute(self, opt, args):
+    """Perform the action, after option parsing is complete.
+    """
+    raise NotImplementedError
+
+  def _ResetPathToProjectMap(self, projects):
+    self._by_path = dict((p.worktree, p) for p in projects)
+
+  def _UpdatePathToProjectMap(self, project):
+    self._by_path[project.worktree] = project
+
+  def _GetProjectByPath(self, manifest, path):
+    project = None
+    if os.path.exists(path):
+      oldpath = None
+      while path and \
+            path != oldpath and \
+            path != manifest.topdir:
+        try:
+          project = self._by_path[path]
+          break
+        except KeyError:
+          oldpath = path
+          path = os.path.dirname(path)
+      if not project and path == manifest.topdir:
+        try:
+          project = self._by_path[path]
+        except KeyError:
+          pass
+    else:
+      try:
+        project = self._by_path[path]
+      except KeyError:
+        pass
+    return project
+
+  def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+                  submodules_ok=False):
+    """A list of projects that match the arguments.
+    """
+    if not manifest:
+      manifest = self.manifest
+    all_projects_list = manifest.projects
+    result = []
+
+    mp = manifest.manifestProject
+
+    if not groups:
+      groups = mp.config.GetString('manifest.groups')
+    if not groups:
+      groups = 'default,platform-' + platform.system().lower()
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+
+    if not args:
+      derived_projects = {}
+      for project in all_projects_list:
+        if submodules_ok or project.sync_s:
+          derived_projects.update((p.name, p)
+                                  for p in project.GetDerivedSubprojects())
+      all_projects_list.extend(derived_projects.values())
+      for project in all_projects_list:
+        if (missing_ok or project.Exists) and project.MatchesGroups(groups):
+          result.append(project)
+    else:
+      self._ResetPathToProjectMap(all_projects_list)
+
+      for arg in args:
+        # We have to filter by manifest groups in case the requested project is
+        # checked out multiple times or differently based on them.
+        projects = [project for project in manifest.GetProjectsWithName(arg)
+                    if project.MatchesGroups(groups)]
+
+        if not projects:
+          path = os.path.abspath(arg).replace('\\', '/')
+          project = self._GetProjectByPath(manifest, path)
+
+          # If it's not a derived project, update path->project mapping and
+          # search again, as arg might actually point to a derived subproject.
+          if (project and not project.Derived and (submodules_ok or
+                                                   project.sync_s)):
+            search_again = False
+            for subproject in project.GetDerivedSubprojects():
+              self._UpdatePathToProjectMap(subproject)
+              search_again = True
+            if search_again:
+              project = self._GetProjectByPath(manifest, path) or project
+
+          if project:
+            projects = [project]
+
+        if not projects:
+          raise NoSuchProjectError(arg)
+
+        for project in projects:
+          if not missing_ok and not project.Exists:
+            raise NoSuchProjectError('%s (%s)' % (arg, project.relpath))
+          if not project.MatchesGroups(groups):
+            raise InvalidProjectGroupsError(arg)
+
+        result.extend(projects)
+
+    def _getpath(x):
+      return x.relpath
+    result.sort(key=_getpath)
+    return result
+
+  def FindProjects(self, args, inverse=False):
+    result = []
+    patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
+    for project in self.GetProjects(''):
+      for pattern in patterns:
+        match = pattern.search(project.name) or pattern.search(project.relpath)
+        if not inverse and match:
+          result.append(project)
+          break
+        if inverse and match:
+          break
+      else:
+        if inverse:
+          result.append(project)
+    result.sort(key=lambda project: project.relpath)
+    return result
+
+
+class InteractiveCommand(Command):
+  """Command which requires user interaction on the tty and
+     must not run within a pager, even if the user asks to.
+  """
+  def WantPager(self, _opt):
+    return False
+
+
+class PagedCommand(Command):
+  """Command which defaults to output in a pager, as its
+     display tends to be larger than one screen full.
+  """
+  def WantPager(self, _opt):
+    return True
+
+
+class MirrorSafeCommand(object):
+  """Command permits itself to run within a mirror,
+     and does not require a working directory.
+  """
+
+
+class GitcAvailableCommand(object):
+  """Command that requires GITC to be available, but does
+     not require the local client to be a GITC client.
+  """
+
+
+class GitcClientCommand(object):
+  """Command that requires the local client to be a GITC
+     client.
+  """

editor.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import re
+import sys
+import subprocess
+import tempfile
+
+from error import EditorError
+import platform_utils
+
+class Editor(object):
+  """Manages the user's preferred text editor."""
+
+  _editor = None
+  globalConfig = None
+
+  @classmethod
+  def _GetEditor(cls):
+    if cls._editor is None:
+      cls._editor = cls._SelectEditor()
+    return cls._editor
+
+  @classmethod
+  def _SelectEditor(cls):
+    e = os.getenv('GIT_EDITOR')
+    if e:
+      return e
+
+    if cls.globalConfig:
+      e = cls.globalConfig.GetString('core.editor')
+      if e:
+        return e
+
+    e = os.getenv('VISUAL')
+    if e:
+      return e
+
+    e = os.getenv('EDITOR')
+    if e:
+      return e
+
+    if os.getenv('TERM') == 'dumb':
+      print(
+"""No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
+Tried to fall back to vi but terminal is dumb.  Please configure at
+least one of these before using this command.""", file=sys.stderr)
+      sys.exit(1)
+
+    return 'vi'
+
+  @classmethod
+  def EditString(cls, data):
+    """Opens an editor to edit the given content.
+
+       Args:
+         data        : the text to edit
+
+      Returns:
+        new value of edited text; None if editing did not succeed
+    """
+    editor = cls._GetEditor()
+    if editor == ':':
+      return data
+
+    fd, path = tempfile.mkstemp()
+    try:
+      os.write(fd, data)
+      os.close(fd)
+      fd = None
+
+      if platform_utils.isWindows():
+        # Split on spaces, respecting quoted strings
+        import shlex
+        args = shlex.split(editor)
+        shell = False
+      elif re.compile("^.*[$ \t'].*$").match(editor):
+        args = [editor + ' "$@"', 'sh']
+        shell = True
+      else:
+        args = [editor]
+        shell = False
+      args.append(path)
+
+      try:
+        rc = subprocess.Popen(args, shell=shell).wait()
+      except OSError as e:
+        raise EditorError('editor failed, %s: %s %s'
+          % (str(e), editor, path))
+      if rc != 0:
+        raise EditorError('editor failed with exit status %d: %s %s'
+          % (rc, editor, path))
+
+      fd2 = open(path)
+      try:
+        return fd2.read()
+      finally:
+        fd2.close()
+    finally:
+      if fd:
+        os.close(fd)
+      platform_utils.remove(path)

error.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class ManifestParseError(Exception):
+  """Failed to parse the manifest file.
+  """
+
+class ManifestInvalidRevisionError(Exception):
+  """The revision value in a project is incorrect.
+  """
+
+class NoManifestException(Exception):
+  """The required manifest does not exist.
+  """
+  def __init__(self, path, reason):
+    super(NoManifestException, self).__init__()
+    self.path = path
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
+class EditorError(Exception):
+  """Unspecified error from the user's text editor.
+  """
+  def __init__(self, reason):
+    super(EditorError, self).__init__()
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
+class GitError(Exception):
+  """Unspecified internal error from git.
+  """
+  def __init__(self, command):
+    super(GitError, self).__init__()
+    self.command = command
+
+  def __str__(self):
+    return self.command
+
+class UploadError(Exception):
+  """A bundle upload to Gerrit did not succeed.
+  """
+  def __init__(self, reason):
+    super(UploadError, self).__init__()
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
+class DownloadError(Exception):
+  """Cannot download a repository.
+  """
+  def __init__(self, reason):
+    super(DownloadError, self).__init__()
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
+class NoSuchProjectError(Exception):
+  """A specified project does not exist in the work tree.
+  """
+  def __init__(self, name=None):
+    super(NoSuchProjectError, self).__init__()
+    self.name = name
+
+  def __str__(self):
+    if self.name is None:
+      return 'in current directory'
+    return self.name
+
+
+class InvalidProjectGroupsError(Exception):
+  """A specified project is not suitable for the specified groups
+  """
+  def __init__(self, name=None):
+    super(InvalidProjectGroupsError, self).__init__()
+    self.name = name
+
+  def __str__(self):
+    if self.name is None:
+      return 'in current directory'
+    return self.name
+
+class RepoChangedException(Exception):
+  """Thrown if 'repo sync' results in repo updating its internal
+     repo or manifest repositories.  In this special case we must
+     use exec to re-execute repo with the new code and manifest.
+  """
+  def __init__(self, extra_args=None):
+    super(RepoChangedException, self).__init__()
+    self.extra_args = extra_args or []
+
+class HookError(Exception):
+  """Thrown if a 'repo-hook' could not be run.
+
+  The common case is that the file wasn't present when we tried to run it.
+  """

event_log.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import json
+import multiprocessing
+
+TASK_COMMAND = 'command'
+TASK_SYNC_NETWORK = 'sync-network'
+TASK_SYNC_LOCAL = 'sync-local'
+
+class EventLog(object):
+  """Event log that records events that occurred during a repo invocation.
+
+  Events are written to the log as a consecutive JSON entries, one per line.
+  Each entry contains the following keys:
+  - id: A ('RepoOp', ID) tuple, suitable for storing in a datastore.
+        The ID is only unique for the invocation of the repo command.
+  - name: Name of the object being operated upon.
+  - task_name: The task that was performed.
+  - start: Timestamp of when the operation started.
+  - finish: Timestamp of when the operation finished.
+  - success: Boolean indicating if the operation was successful.
+  - try_count: A counter indicating the try count of this task.
+
+  Optionally:
+  - parent: A ('RepoOp', ID) tuple indicating the parent event for nested
+            events.
+
+  Valid task_names include:
+  - command: The invocation of a subcommand.
+  - sync-network: The network component of a sync command.
+  - sync-local: The local component of a sync command.
+
+  Specific tasks may include additional informational properties.
+  """
+
+  def __init__(self):
+    """Initializes the event log."""
+    self._log = []
+    self._parent = None
+
+  def Add(self, name, task_name, start, finish=None, success=None,
+          try_count=1, kind='RepoOp'):
+    """Add an event to the log.
+
+    Args:
+      name: Name of the object being operated upon.
+      task_name: A sub-task that was performed for name.
+      start: Timestamp of when the operation started.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+      try_count: A counter indicating the try count of this task.
+      kind: The kind of the object for the unique identifier.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event = {
+        'id': (kind, _NextEventId()),
+        'name': name,
+        'task_name': task_name,
+        'start_time': start,
+        'try': try_count,
+    }
+
+    if self._parent:
+      event['parent'] = self._parent['id']
+
+    if success is not None or finish is not None:
+        self.FinishEvent(event, finish, success)
+
+    self._log.append(event)
+    return event
+
+  def AddSync(self, project, task_name, start, finish, success):
+    """Add a event to the log for a sync command.
+
+    Args:
+      project: Project being synced.
+      task_name: A sub-task that was performed for name.
+                 One of (TASK_SYNC_NETWORK, TASK_SYNC_LOCAL)
+      start: Timestamp of when the operation started.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event = self.Add(project.relpath, task_name, start, finish, success)
+    if event is not None:
+      event['project'] = project.name
+      if project.revisionExpr:
+        event['revision'] = project.revisionExpr
+      if project.remote.url:
+        event['project_url'] = project.remote.url
+      if project.remote.fetchUrl:
+        event['remote_url'] = project.remote.fetchUrl
+      try:
+        event['git_hash'] = project.GetCommitRevisionId()
+      except Exception:
+        pass
+    return event
+
+  def GetStatusString(self, success):
+    """Converst a boolean success to a status string.
+
+    Args:
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      status string.
+    """
+    return 'pass' if success else 'fail'
+
+  def FinishEvent(self, event, finish, success):
+    """Finishes an incomplete event.
+
+    Args:
+      event: An event that has been added to the log.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event['status'] =  self.GetStatusString(success)
+    event['finish_time'] = finish
+    return event
+
+  def SetParent(self, event):
+    """Set a parent event for all new entities.
+
+    Args:
+      event: The event to use as a parent.
+    """
+    self._parent = event
+
+  def Write(self, filename):
+    """Writes the log out to a file.
+
+    Args:
+      filename: The file to write the log to.
+    """
+    with open(filename, 'w+') as f:
+      for e in self._log:
+        json.dump(e, f, sort_keys=True)
+        f.write('\n')
+
+
+# An integer id that is unique across this invocation of the program.
+_EVENT_ID = multiprocessing.Value('i', 1)
+
+def _NextEventId():
+  """Helper function for grabbing the next unique id.
+
+  Returns:
+    A unique, to this invocation of the program, integer id.
+  """
+  with _EVENT_ID.get_lock():
+    val = _EVENT_ID.value
+    _EVENT_ID.value += 1
+  return val

git_command.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import tempfile
+from signal import SIGTERM
+
+from error import GitError
+from git_refs import HEAD
+import platform_utils
+from repo_trace import REPO_TRACE, IsTrace, Trace
+from wrapper import Wrapper
+
+GIT = 'git'
+MIN_GIT_VERSION = (1, 5, 4)
+GIT_DIR = 'GIT_DIR'
+
+LAST_GITDIR = None
+LAST_CWD = None
+
+_ssh_proxy_path = None
+_ssh_sock_path = None
+_ssh_clients = []
+
+def ssh_sock(create=True):
+  global _ssh_sock_path
+  if _ssh_sock_path is None:
+    if not create:
+      return None
+    tmp_dir = '/tmp'
+    if not os.path.exists(tmp_dir):
+      tmp_dir = tempfile.gettempdir()
+    _ssh_sock_path = os.path.join(
+      tempfile.mkdtemp('', 'ssh-', tmp_dir),
+      'master-%r@%h:%p')
+  return _ssh_sock_path
+
+def _ssh_proxy():
+  global _ssh_proxy_path
+  if _ssh_proxy_path is None:
+    _ssh_proxy_path = os.path.join(
+      os.path.dirname(__file__),
+      'git_ssh')
+  return _ssh_proxy_path
+
+def _add_ssh_client(p):
+  _ssh_clients.append(p)
+
+def _remove_ssh_client(p):
+  try:
+    _ssh_clients.remove(p)
+  except ValueError:
+    pass
+
+def terminate_ssh_clients():
+  global _ssh_clients
+  for p in _ssh_clients:
+    try:
+      os.kill(p.pid, SIGTERM)
+      p.wait()
+    except OSError:
+      pass
+  _ssh_clients = []
+
+_git_version = None
+
+class _GitCall(object):
+  def version_tuple(self):
+    global _git_version
+    if _git_version is None:
+      _git_version = Wrapper().ParseGitVersion()
+      if _git_version is None:
+        print('fatal: unable to detect git version', file=sys.stderr)
+        sys.exit(1)
+    return _git_version
+
+  def __getattr__(self, name):
+    name = name.replace('_','-')
+    def fun(*cmdv):
+      command = [name]
+      command.extend(cmdv)
+      return GitCommand(None, command).Wait() == 0
+    return fun
+git = _GitCall()
+
+
+def RepoSourceVersion():
+  """Return the version of the repo.git tree."""
+  ver = getattr(RepoSourceVersion, 'version', None)
+
+  # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
+  # to initialize version info we provide.
+  if ver is None:
+    env = GitCommand._GetBasicEnv()
+
+    proj = os.path.dirname(os.path.abspath(__file__))
+    env[GIT_DIR] = os.path.join(proj, '.git')
+
+    p = subprocess.Popen([GIT, 'describe', HEAD], stdout=subprocess.PIPE,
+                         env=env)
+    if p.wait() == 0:
+      ver = p.stdout.read().strip().decode('utf-8')
+      if ver.startswith('v'):
+        ver = ver[1:]
+    else:
+      ver = 'unknown'
+    setattr(RepoSourceVersion, 'version', ver)
+
+  return ver
+
+
+class UserAgent(object):
+  """Mange User-Agent settings when talking to external services
+
+  We follow the style as documented here:
+  https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
+  """
+
+  _os = None
+  _repo_ua = None
+  _git_ua = None
+
+  @property
+  def os(self):
+    """The operating system name."""
+    if self._os is None:
+      os_name = sys.platform
+      if os_name.lower().startswith('linux'):
+        os_name = 'Linux'
+      elif os_name == 'win32':
+        os_name = 'Win32'
+      elif os_name == 'cygwin':
+        os_name = 'Cygwin'
+      elif os_name == 'darwin':
+        os_name = 'Darwin'
+      self._os = os_name
+
+    return self._os
+
+  @property
+  def repo(self):
+    """The UA when connecting directly from repo."""
+    if self._repo_ua is None:
+      py_version = sys.version_info
+      self._repo_ua = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
+          RepoSourceVersion(),
+          self.os,
+          git.version_tuple().full,
+          py_version.major, py_version.minor, py_version.micro)
+
+    return self._repo_ua
+
+  @property
+  def git(self):
+    """The UA when running git."""
+    if self._git_ua is None:
+      self._git_ua = 'git/%s (%s) git-repo/%s' % (
+          git.version_tuple().full,
+          self.os,
+          RepoSourceVersion())
+
+    return self._git_ua
+
+user_agent = UserAgent()
+
+def git_require(min_version, fail=False, msg=''):
+  git_version = git.version_tuple()
+  if min_version <= git_version:
+    return True
+  if fail:
+    need = '.'.join(map(str, min_version))
+    if msg:
+      msg = ' for ' + msg
+    print('fatal: git %s or later required%s' % (need, msg), file=sys.stderr)
+    sys.exit(1)
+  return False
+
+def _setenv(env, name, value):
+  env[name] = value.encode()
+
+class GitCommand(object):
+  def __init__(self,
+               project,
+               cmdv,
+               bare = False,
+               provide_stdin = False,
+               capture_stdout = False,
+               capture_stderr = False,
+               disable_editor = False,
+               ssh_proxy = False,
+               cwd = None,
+               gitdir = None):
+    env = self._GetBasicEnv()
+
+    # If we are not capturing std* then need to print it.
+    self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr}
+
+    if disable_editor:
+      _setenv(env, 'GIT_EDITOR', ':')
+    if ssh_proxy:
+      _setenv(env, 'REPO_SSH_SOCK', ssh_sock())
+      _setenv(env, 'GIT_SSH', _ssh_proxy())
+      _setenv(env, 'GIT_SSH_VARIANT', 'ssh')
+    if 'http_proxy' in env and 'darwin' == sys.platform:
+      s = "'http.proxy=%s'" % (env['http_proxy'],)
+      p = env.get('GIT_CONFIG_PARAMETERS')
+      if p is not None:
+        s = p + ' ' + s
+      _setenv(env, 'GIT_CONFIG_PARAMETERS', s)
+    if 'GIT_ALLOW_PROTOCOL' not in env:
+      _setenv(env, 'GIT_ALLOW_PROTOCOL',
+              'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
+    _setenv(env, 'GIT_HTTP_USER_AGENT', user_agent.git)
+
+    if project:
+      if not cwd:
+        cwd = project.worktree
+      if not gitdir:
+        gitdir = project.gitdir
+
+    command = [GIT]
+    if bare:
+      if gitdir:
+        _setenv(env, GIT_DIR, gitdir)
+      cwd = None
+    command.append(cmdv[0])
+    # Need to use the --progress flag for fetch/clone so output will be
+    # displayed as by default git only does progress output if stderr is a TTY.
+    if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'):
+      if '--progress' not in cmdv and '--quiet' not in cmdv:
+        command.append('--progress')
+    command.extend(cmdv[1:])
+
+    if provide_stdin:
+      stdin = subprocess.PIPE
+    else:
+      stdin = None
+
+    stdout = subprocess.PIPE
+    stderr = subprocess.PIPE
+
+    if IsTrace():
+      global LAST_CWD
+      global LAST_GITDIR
+
+      dbg = ''
+
+      if cwd and LAST_CWD != cwd:
+        if LAST_GITDIR or LAST_CWD:
+          dbg += '\n'
+        dbg += ': cd %s\n' % cwd
+        LAST_CWD = cwd
+
+      if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]:
+        if LAST_GITDIR or LAST_CWD:
+          dbg += '\n'
+        dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
+        LAST_GITDIR = env[GIT_DIR]
+
+      dbg += ': '
+      dbg += ' '.join(command)
+      if stdin == subprocess.PIPE:
+        dbg += ' 0<|'
+      if stdout == subprocess.PIPE:
+        dbg += ' 1>|'
+      if stderr == subprocess.PIPE:
+        dbg += ' 2>|'
+      Trace('%s', dbg)
+
+    try:
+      p = subprocess.Popen(command,
+                           cwd = cwd,
+                           env = env,
+                           stdin = stdin,
+                           stdout = stdout,
+                           stderr = stderr)
+    except Exception as e:
+      raise GitError('%s: %s' % (command[1], e))
+
+    if ssh_proxy:
+      _add_ssh_client(p)
+
+    self.process = p
+    self.stdin = p.stdin
+
+  @staticmethod
+  def _GetBasicEnv():
+    """Return a basic env for running git under.
+
+    This is guaranteed to be side-effect free.
+    """
+    env = os.environ.copy()
+    for key in (REPO_TRACE,
+                GIT_DIR,
+                'GIT_ALTERNATE_OBJECT_DIRECTORIES',
+                'GIT_OBJECT_DIRECTORY',
+                'GIT_WORK_TREE',
+                'GIT_GRAFT_FILE',
+                'GIT_INDEX_FILE'):
+      env.pop(key, None)
+    return env
+
+  def Wait(self):
+    try:
+      p = self.process
+      rc = self._CaptureOutput()
+    finally:
+      _remove_ssh_client(p)
+    return rc
+
+  def _CaptureOutput(self):
+    p = self.process
+    s_in = platform_utils.FileDescriptorStreams.create()
+    s_in.add(p.stdout, sys.stdout, 'stdout')
+    s_in.add(p.stderr, sys.stderr, 'stderr')
+    self.stdout = ''
+    self.stderr = ''
+
+    while not s_in.is_done:
+      in_ready = s_in.select()
+      for s in in_ready:
+        buf = s.read()
+        if not buf:
+          s_in.remove(s)
+          continue
+        if not hasattr(buf, 'encode'):
+          buf = buf.decode()
+        if s.std_name == 'stdout':
+          self.stdout += buf
+        else:
+          self.stderr += buf
+        if self.tee[s.std_name]:
+          s.dest.write(buf)
+          s.dest.flush()
+    return p.wait()

git_refs.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from repo_trace import Trace
+import platform_utils
+
+HEAD      = 'HEAD'
+R_CHANGES = 'refs/changes/'
+R_HEADS   = 'refs/heads/'
+R_TAGS    = 'refs/tags/'
+R_PUB     = 'refs/published/'
+R_M       = 'refs/remotes/m/'
+
+
+class GitRefs(object):
+  def __init__(self, gitdir):
+    self._gitdir = gitdir
+    self._phyref = None
+    self._symref = None
+    self._mtime = {}
+
+  @property
+  def all(self):
+    self._EnsureLoaded()
+    return self._phyref
+
+  def get(self, name):
+    try:
+      return self.all[name]
+    except KeyError:
+      return ''
+
+  def deleted(self, name):
+    if self._phyref is not None:
+      if name in self._phyref:
+        del self._phyref[name]
+
+      if name in self._symref:
+        del self._symref[name]
+
+      if name in self._mtime:
+        del self._mtime[name]
+
+  def symref(self, name):
+    try:
+      self._EnsureLoaded()
+      return self._symref[name]
+    except KeyError:
+      return ''
+
+  def _EnsureLoaded(self):
+    if self._phyref is None or self._NeedUpdate():
+      self._LoadAll()
+
+  def _NeedUpdate(self):
+    Trace(': scan refs %s', self._gitdir)
+
+    for name, mtime in self._mtime.items():
+      try:
+        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+          return True
+      except OSError:
+        return True
+    return False
+
+  def _LoadAll(self):
+    Trace(': load refs %s', self._gitdir)
+
+    self._phyref = {}
+    self._symref = {}
+    self._mtime = {}
+
+    self._ReadPackedRefs()
+    self._ReadLoose('refs/')
+    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+
+    scan = self._symref
+    attempts = 0
+    while scan and attempts < 5:
+      scan_next = {}
+      for name, dest in scan.items():
+        if dest in self._phyref:
+          self._phyref[name] = self._phyref[dest]
+        else:
+          scan_next[name] = dest
+      scan = scan_next
+      attempts += 1
+
+  def _ReadPackedRefs(self):
+    path = os.path.join(self._gitdir, 'packed-refs')
+    try:
+      fd = open(path, 'r')
+      mtime = os.path.getmtime(path)
+    except IOError:
+      return
+    except OSError:
+      return
+    try:
+      for line in fd:
+        line = str(line)
+        if line[0] == '#':
+          continue
+        if line[0] == '^':
+          continue
+
+        line = line[:-1]
+        p = line.split(' ')
+        ref_id = p[0]
+        name = p[1]
+
+        self._phyref[name] = ref_id
+    finally:
+      fd.close()
+    self._mtime['packed-refs'] = mtime
+
+  def _ReadLoose(self, prefix):
+    base = os.path.join(self._gitdir, prefix)
+    for name in platform_utils.listdir(base):
+      p = os.path.join(base, name)
+      if platform_utils.isdir(p):
+        self._mtime[prefix] = os.path.getmtime(base)
+        self._ReadLoose(prefix + name + '/')
+      elif name.endswith('.lock'):
+        pass
+      else:
+        self._ReadLoose1(p, prefix + name)
+
+  def _ReadLoose1(self, path, name):
+    try:
+      fd = open(path)
+    except IOError:
+      return
+
+    try:
+      try:
+        mtime = os.path.getmtime(path)
+        ref_id = fd.readline()
+      except (IOError, OSError):
+        return
+    finally:
+      fd.close()
+
+    try:
+      ref_id = ref_id.decode()
+    except AttributeError:
+      pass
+    if not ref_id:
+      return
+    ref_id = ref_id[:-1]
+
+    if ref_id.startswith('ref: '):
+      self._symref[name] = ref_id[5:]
+    else:
+      self._phyref[name] = ref_id
+    self._mtime[name] = mtime

git_ssh

+#!/bin/sh
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exec ssh -o "ControlMaster no" -o "ControlPath $REPO_SSH_SOCK" "$@"

gitc_utils.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import platform
+import re
+import sys
+import time
+
+import git_command
+import git_config
+import wrapper
+
+from error import ManifestParseError
+
+NUM_BATCH_RETRIEVE_REVISIONID = 32
+
+def get_gitc_manifest_dir():
+  return wrapper.Wrapper().get_gitc_manifest_dir()
+
+def parse_clientdir(gitc_fs_path):
+  return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
+
+def _set_project_revisions(projects):
+  """Sets the revisionExpr for a list of projects.
+
+  Because of the limit of open file descriptors allowed, length of projects
+  should not be overly large. Recommend calling this function multiple times
+  with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+  @param projects: List of project objects to set the revionExpr for.
+  """
+  # Retrieve the commit id for each project based off of it's current
+  # revisionExpr and it is not already a commit id.
+  project_gitcmds = [(
+      project, git_command.GitCommand(None,
+                                      ['ls-remote',
+                                       project.remote.url,
+                                       project.revisionExpr],
+                                      capture_stdout=True, cwd='/tmp'))
+      for project in projects if not git_config.IsId(project.revisionExpr)]
+  for proj, gitcmd in project_gitcmds:
+    if gitcmd.Wait():
+      print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
+      sys.exit(1)
+    revisionExpr = gitcmd.stdout.split('\t')[0]
+    if not revisionExpr:
+      raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
+                               (proj.remote.url, proj.revisionExpr))
+    proj.revisionExpr = revisionExpr
+
+def _manifest_groups(manifest):
+  """Returns the manifest group string that should be synced
+
+  This is the same logic used by Command.GetProjects(), which is used during
+  repo sync
+
+  @param manifest: The XmlManifest object
+  """
+  mp = manifest.manifestProject
+  groups = mp.config.GetString('manifest.groups')
+  if not groups:
+    groups = 'default,platform-' + platform.system().lower()
+  return groups
+
+def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
+  """Generate a manifest for shafsd to use for this GITC client.
+
+  @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
+  @param manifest: A GitcManifest object loaded with the current repo manifest.
+  @param paths: List of project paths we want to update.
+  """
+
+  print('Generating GITC Manifest by fetching revision SHAs for each '
+        'project.')
+  if paths is None:
+    paths = list(manifest.paths.keys())
+
+  groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
+
+  # Convert the paths to projects, and filter them to the matched groups.
+  projects = [manifest.paths[p] for p in paths]
+  projects = [p for p in projects if p.MatchesGroups(groups)]
+
+  if gitc_manifest is not None:
+    for path, proj in manifest.paths.items():
+      if not proj.MatchesGroups(groups):
+        continue
+
+      if not proj.upstream and not git_config.IsId(proj.revisionExpr):
+        proj.upstream = proj.revisionExpr
+
+      if not path in gitc_manifest.paths:
+        # Any new projects need their first revision, even if we weren't asked
+        # for them.
+        projects.append(proj)
+      elif not path in paths:
+        # And copy revisions from the previous manifest if we're not updating
+        # them now.
+        gitc_proj = gitc_manifest.paths[path]
+        if gitc_proj.old_revision:
+          proj.revisionExpr = None
+          proj.old_revision = gitc_proj.old_revision
+        else:
+          proj.revisionExpr = gitc_proj.revisionExpr
+
+  index = 0
+  while index < len(projects):
+    _set_project_revisions(
+        projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+    index += NUM_BATCH_RETRIEVE_REVISIONID
+
+  if gitc_manifest is not None:
+    for path, proj in gitc_manifest.paths.items():
+      if proj.old_revision and path in paths:
+        # If we updated a project that has been started, keep the old-revision
+        # updated.
+        repo_proj = manifest.paths[path]
+        repo_proj.old_revision = repo_proj.revisionExpr
+        repo_proj.revisionExpr = None
+
+  # Convert URLs from relative to absolute.
+  for _name, remote in manifest.remotes.items():
+    remote.fetchUrl = remote.resolvedFetchUrl
+
+  # Save the manifest.
+  save_manifest(manifest)
+
+def save_manifest(manifest, client_dir=None):
+  """Save the manifest file in the client_dir.
+
+  @param client_dir: Client directory to save the manifest in.
+  @param manifest: Manifest object to save.
+  """
+  if not client_dir:
+    client_dir = manifest.gitc_client_dir
+  with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+    manifest.Save(f, groups=_manifest_groups(manifest))
+  # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
+  # Give the GITC filesystem time to register the manifest changes.
+  time.sleep(3)

pager.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import select
+import subprocess
+import sys
+
+import platform_utils
+
+active = False
+pager_process = None
+old_stdout = None
+old_stderr = None
+
+def RunPager(globalConfig):
+  if not os.isatty(0) or not os.isatty(1):
+    return
+  pager = _SelectPager(globalConfig)
+  if pager == '' or pager == 'cat':
+    return
+
+  if platform_utils.isWindows():
+    _PipePager(pager);
+  else:
+    _ForkPager(pager)
+
+def TerminatePager():
+  global pager_process, old_stdout, old_stderr
+  if pager_process:
+    sys.stdout.flush()
+    sys.stderr.flush()
+    pager_process.stdin.close()
+    pager_process.wait();
+    pager_process = None
+    # Restore initial stdout/err in case there is more output in this process
+    # after shutting down the pager process
+    sys.stdout = old_stdout
+    sys.stderr = old_stderr
+
+def _PipePager(pager):
+  global pager_process, old_stdout, old_stderr
+  assert pager_process is None, "Only one active pager process at a time"
+  # Create pager process, piping stdout/err into its stdin
+  pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout, stderr=sys.stderr)
+  old_stdout = sys.stdout
+  old_stderr = sys.stderr
+  sys.stdout = pager_process.stdin
+  sys.stderr = pager_process.stdin
+
+def _ForkPager(pager):
+  global active
+  # This process turns into the pager; a child it forks will
+  # do the real processing and output back to the pager. This
+  # is necessary to keep the pager in control of the tty.
+  #
+  try:
+    r, w = os.pipe()
+    pid = os.fork()
+    if not pid:
+      os.dup2(w, 1)
+      os.dup2(w, 2)
+      os.close(r)
+      os.close(w)
+      active = True
+      return
+
+    os.dup2(r, 0)
+    os.close(r)
+    os.close(w)
+
+    _BecomePager(pager)
+  except Exception:
+    print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
+    sys.exit(255)
+
+def _SelectPager(globalConfig):
+  try:
+    return os.environ['GIT_PAGER']
+  except KeyError:
+    pass
+
+  pager = globalConfig.GetString('core.pager')
+  if pager:
+    return pager
+
+  try:
+    return os.environ['PAGER']
+  except KeyError:
+    pass
+
+  return 'less'
+
+def _BecomePager(pager):
+  # Delaying execution of the pager until we have output
+  # ready works around a long-standing bug in popularly
+  # available versions of 'less', a better 'more'.
+  #
+  _a, _b, _c = select.select([0], [], [0])
+
+  os.environ['LESS'] = 'FRSX'
+
+  try:
+    os.execvp(pager, [pager])
+  except OSError:
+    os.execv('/bin/sh', ['sh', '-c', pager])

platform_utils_win32.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+
+from ctypes import WinDLL, get_last_error, FormatError, WinError, addressof
+from ctypes import c_buffer
+from ctypes.wintypes import BOOL, BOOLEAN, LPCWSTR, DWORD, HANDLE, POINTER, c_ubyte
+from ctypes.wintypes import WCHAR, USHORT, LPVOID, Structure, Union, ULONG
+from ctypes.wintypes import byref
+
+kernel32 = WinDLL('kernel32', use_last_error=True)
+
+LPDWORD = POINTER(DWORD)
+UCHAR = c_ubyte
+
+# Win32 error codes
+ERROR_SUCCESS = 0
+ERROR_NOT_SUPPORTED = 50
+ERROR_PRIVILEGE_NOT_HELD = 1314
+
+# Win32 API entry points
+CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
+CreateSymbolicLinkW.restype = BOOLEAN
+CreateSymbolicLinkW.argtypes = (LPCWSTR,  # lpSymlinkFileName In
+                                LPCWSTR,  # lpTargetFileName In
+                                DWORD)    # dwFlags In
+
+# Symbolic link creation flags
+SYMBOLIC_LINK_FLAG_FILE = 0x00
+SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
+# symlink support for CreateSymbolicLink() starting with Windows 10 (1703, v10.0.14972)
+SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 0x02
+
+GetFileAttributesW = kernel32.GetFileAttributesW
+GetFileAttributesW.restype = DWORD
+GetFileAttributesW.argtypes = (LPCWSTR,)  # lpFileName In
+
+INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
+FILE_ATTRIBUTE_REPARSE_POINT = 0x00400
+
+CreateFileW = kernel32.CreateFileW
+CreateFileW.restype = HANDLE
+CreateFileW.argtypes = (LPCWSTR,  # lpFileName In
+                        DWORD,    # dwDesiredAccess In
+                        DWORD,    # dwShareMode In
+                        LPVOID,   # lpSecurityAttributes In_opt
+                        DWORD,    # dwCreationDisposition In
+                        DWORD,    # dwFlagsAndAttributes In
+                        HANDLE)   # hTemplateFile In_opt
+
+CloseHandle = kernel32.CloseHandle
+CloseHandle.restype = BOOL
+CloseHandle.argtypes = (HANDLE,)  # hObject In
+
+INVALID_HANDLE_VALUE = HANDLE(-1).value
+OPEN_EXISTING = 3
+FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+
+DeviceIoControl = kernel32.DeviceIoControl
+DeviceIoControl.restype = BOOL
+DeviceIoControl.argtypes = (HANDLE,   # hDevice In
+                            DWORD,    # dwIoControlCode In
+                            LPVOID,   # lpInBuffer In_opt
+                            DWORD,    # nInBufferSize In
+                            LPVOID,   # lpOutBuffer Out_opt
+                            DWORD,    # nOutBufferSize In
+                            LPDWORD,  # lpBytesReturned Out_opt
+                            LPVOID)   # lpOverlapped Inout_opt
+
+# Device I/O control flags and options
+FSCTL_GET_REPARSE_POINT = 0x000900A8
+IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003
+IO_REPARSE_TAG_SYMLINK = 0xA000000C
+MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 0x4000
+
+
+class GENERIC_REPARSE_BUFFER(Structure):
+  _fields_ = (('DataBuffer', UCHAR * 1),)
+
+
+class SYMBOLIC_LINK_REPARSE_BUFFER(Structure):
+  _fields_ = (('SubstituteNameOffset', USHORT),
+              ('SubstituteNameLength', USHORT),
+              ('PrintNameOffset', USHORT),
+              ('PrintNameLength', USHORT),
+              ('Flags', ULONG),
+              ('PathBuffer', WCHAR * 1))
+
+  @property
+  def PrintName(self):
+    arrayt = WCHAR * (self.PrintNameLength // 2)
+    offset = type(self).PathBuffer.offset + self.PrintNameOffset
+    return arrayt.from_address(addressof(self) + offset).value
+
+
+class MOUNT_POINT_REPARSE_BUFFER(Structure):
+  _fields_ = (('SubstituteNameOffset', USHORT),
+              ('SubstituteNameLength', USHORT),
+              ('PrintNameOffset', USHORT),
+              ('PrintNameLength', USHORT),
+              ('PathBuffer', WCHAR * 1))
+
+  @property
+  def PrintName(self):
+    arrayt = WCHAR * (self.PrintNameLength // 2)
+    offset = type(self).PathBuffer.offset + self.PrintNameOffset
+    return arrayt.from_address(addressof(self) + offset).value
+
+
+class REPARSE_DATA_BUFFER(Structure):
+  class REPARSE_BUFFER(Union):
+    _fields_ = (('SymbolicLinkReparseBuffer', SYMBOLIC_LINK_REPARSE_BUFFER),
+                ('MountPointReparseBuffer', MOUNT_POINT_REPARSE_BUFFER),
+                ('GenericReparseBuffer', GENERIC_REPARSE_BUFFER))
+  _fields_ = (('ReparseTag', ULONG),
+              ('ReparseDataLength', USHORT),
+              ('Reserved', USHORT),
+              ('ReparseBuffer', REPARSE_BUFFER))
+  _anonymous_ = ('ReparseBuffer',)
+
+
+def create_filesymlink(source, link_name):
+  """Creates a Windows file symbolic link source pointing to link_name."""
+  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
+
+
+def create_dirsymlink(source, link_name):
+  """Creates a Windows directory symbolic link source pointing to link_name.
+  """
+  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
+
+
+def _create_symlink(source, link_name, dwFlags):
+  if not CreateSymbolicLinkW(link_name, source, dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE):
+    # See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0
+    # "the unprivileged create flag is unsupported below Windows 10 (1703, v10.0.14972).
+    # retry without it."
+    if not CreateSymbolicLinkW(link_name, source, dwFlags):
+      code = get_last_error()
+      error_desc = FormatError(code).strip()
+      if code == ERROR_PRIVILEGE_NOT_HELD:
+        raise OSError(errno.EPERM, error_desc, link_name)
+      _raise_winerror(
+          code,
+          'Error creating symbolic link \"%s\"'.format(link_name))
+
+
+def islink(path):
+  result = GetFileAttributesW(path)
+  if result == INVALID_FILE_ATTRIBUTES:
+    return False
+  return bool(result & FILE_ATTRIBUTE_REPARSE_POINT)
+
+
+def readlink(path):
+  reparse_point_handle = CreateFileW(path,
+                                     0,
+                                     0,
+                                     None,
+                                     OPEN_EXISTING,
+                                     FILE_FLAG_OPEN_REPARSE_POINT |
+                                     FILE_FLAG_BACKUP_SEMANTICS,
+                                     None)
+  if reparse_point_handle == INVALID_HANDLE_VALUE:
+    _raise_winerror(
+        get_last_error(),
+        'Error opening symblic link \"%s\"'.format(path))
+  target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
+  n_bytes_returned = DWORD()
+  io_result = DeviceIoControl(reparse_point_handle,
+                              FSCTL_GET_REPARSE_POINT,
+                              None,
+                              0,
+                              target_buffer,
+                              len(target_buffer),
+                              byref(n_bytes_returned),
+                              None)
+  CloseHandle(reparse_point_handle)
+  if not io_result:
+    _raise_winerror(
+        get_last_error(),
+        'Error reading symblic link \"%s\"'.format(path))
+  rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
+  if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
+    return _preserve_encoding(path, rdb.SymbolicLinkReparseBuffer.PrintName)
+  elif rdb.ReparseTag == IO_REPARSE_TAG_MOUNT_POINT:
+    return _preserve_encoding(path, rdb.MountPointReparseBuffer.PrintName)
+  # Unsupported reparse point type
+  _raise_winerror(
+      ERROR_NOT_SUPPORTED,
+      'Error reading symblic link \"%s\"'.format(path))
+
+
+def _preserve_encoding(source, target):
+  """Ensures target is the same string type (i.e. unicode or str) as source."""
+  if isinstance(source, unicode):
+    return unicode(target)
+  return str(target)
+
+
+def _raise_winerror(code, error_desc):
+  win_error_desc = FormatError(code).strip()
+  error_desc = "%s: %s".format(error_desc, win_error_desc)
+  raise WinError(code, error_desc)

progress.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from time import time
+from repo_trace import IsTrace
+
+_NOT_TTY = not os.isatty(2)
+
+# This will erase all content in the current line (wherever the cursor is).
+# It does not move the cursor, so this is usually followed by \r to move to
+# column 0.
+CSI_ERASE_LINE = '\x1b[2K'
+
+class Progress(object):
+  def __init__(self, title, total=0, units='', print_newline=False,
+               always_print_percentage=False):
+    self._title = title
+    self._total = total
+    self._done = 0
+    self._lastp = -1
+    self._start = time()
+    self._show = False
+    self._units = units
+    self._print_newline = print_newline
+    self._always_print_percentage = always_print_percentage
+
+  def update(self, inc=1):
+    self._done += inc
+
+    if _NOT_TTY or IsTrace():
+      return
+
+    if not self._show:
+      if 0.5 <= time() - self._start:
+        self._show = True
+      else:
+        return
+
+    if self._total <= 0:
+      sys.stderr.write('%s\r%s: %d,' % (
+        CSI_ERASE_LINE,
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+
+      if self._lastp != p or self._always_print_percentage:
+        self._lastp = p
+        sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s)%s' % (
+          CSI_ERASE_LINE,
+          self._title,
+          p,
+          self._done, self._units,
+          self._total, self._units,
+          "\n" if self._print_newline else ""))
+        sys.stderr.flush()
+
+  def end(self):
+    if _NOT_TTY or IsTrace() or not self._show:
+      return
+
+    if self._total <= 0:
+      sys.stderr.write('%s\r%s: %d, done.\n' % (
+        CSI_ERASE_LINE,
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+      sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done.\n' % (
+        CSI_ERASE_LINE,
+        self._title,
+        p,
+        self._done, self._units,
+        self._total, self._units))
+      sys.stderr.flush()

pyversion.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2013 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+def is_python3():
+  return sys.version_info[0] == 3

repo/.flake8

+[flake8]
+max-line-length=80
+ignore=E111,E114,E402

repo/.gitattributes

+# Prevent /bin/sh scripts from being clobbered by autocrlf=true
+git_ssh text eol=lf
+repo text eol=lf
+hooks/* text eol=lf

repo/.gitignore

+*.pyc
+.repopickle_*
+/repoc

repo/.mailmap

+Anthony Newnam <anthony.newnam@garmin.com>    Anthony <anthony@bnovc.com>
+He Ping <tdihp@hotmail.com>                   heping <tdihp@hotmail.com>
+Hu Xiuyun <xiuyun.hu@hisilicon.com>           Hu xiuyun <xiuyun.hu@hisilicon.com>
+Hu Xiuyun <xiuyun.hu@hisilicon.com>           Hu Xiuyun <clouds08@qq.com>
+Jelly Chen <chenguodong@huawei.com>           chenguodong <chenguodong@huawei.com>
+Jia Bi <bijia@xiaomi.com>                     bijia <bijia@xiaomi.com>
+JoonCheol Park <jooncheol@gmail.com>          Jooncheol Park <jooncheol@gmail.com>
+Sergii Pylypenko <x.pelya.x@gmail.com>        pelya <x.pelya.x@gmail.com>
+Shawn Pearce <sop@google.com>                 Shawn O. Pearce <sop@google.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjolin <ulrik.sjolin@gmail.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjolin <ulrik.sjolin@sonyericsson.com>
+Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>  Ulrik Sjölin <ulrik.sjolin@sonyericsson.com>

repo/.project

+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>git-repo</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

repo/.pydevproject

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/git-repo</path>
+</pydev_pathproperty>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+</pydev_project>

repo/README.md

+# repo
+
+Repo is a tool built on top of Git.  Repo helps manage many Git repositories,
+does the uploads to revision control systems, and automates parts of the
+development workflow.  Repo is not meant to replace Git, only to make it
+easier to work with Git.  The repo command is an executable Python script
+that you can put anywhere in your path.
+
+* Homepage: https://gerrit.googlesource.com/git-repo/
+* Bug reports: https://bugs.chromium.org/p/gerrit/issues/list?q=component:repo
+* Source: https://gerrit.googlesource.com/git-repo/
+* Overview: https://source.android.com/source/developing.html
+* Docs: https://source.android.com/source/using-repo.html
+* [repo Manifest Format](./docs/manifest-format.md)
+* [repo Hooks](./docs/repo-hooks.md)
+* [Submitting patches](./SUBMITTING_PATCHES.md)

repo/SUBMITTING_PATCHES.md

+[TOC]
+
+# Short Version
+
+ - Make small logical changes.
+ - Provide a meaningful commit message.
+ - Check for coding errors and style nits with pyflakes and flake8
+ - Make sure all code is under the Apache License, 2.0.
+ - Publish your changes for review.
+ - Make corrections if requested.
+ - Verify your changes on gerrit so they can be submitted.
+
+   `git push https://gerrit-review.googlesource.com/git-repo HEAD:refs/for/master`
+
+
+# Long Version
+
+I wanted a file describing how to submit patches for repo,
+so I started with the one found in the core Git distribution
+(Documentation/SubmittingPatches), which itself was based on the
+patch submission guidelines for the Linux kernel.
+
+However there are some differences, so please review and familiarize
+yourself with the following relevant bits.
+
+
+## Make separate commits for logically separate changes.
+
+Unless your patch is really trivial, you should not be sending
+out a patch that was generated between your working tree and your
+commit head.  Instead, always make a commit with complete commit
+message and generate a series of patches from your repository.
+It is a good discipline.
+
+Describe the technical detail of the change(s).
+
+If your description starts to get too long, that's a sign that you
+probably need to split up your commit to finer grained pieces.
+
+
+## Check for coding errors and style nits with pyflakes and flake8
+
+### Coding errors
+
+Run `pyflakes` on changed modules:
+
+    pyflakes file.py
+
+Ideally there should be no new errors or warnings introduced.
+
+### Style violations
+
+Run `flake8` on changes modules:
+
+    flake8 file.py
+
+Note that repo generally follows [Google's python style guide] rather than
+[PEP 8], so it's possible that the output of `flake8` will be quite noisy.
+It's not mandatory to avoid all warnings, but at least the maximum line
+length should be followed.
+
+If there are many occurrences of the same warning that cannot be
+avoided without going against the Google style guide, these may be
+suppressed in the included `.flake8` file.
+
+[Google's python style guide]: https://google.github.io/styleguide/pyguide.html
+[PEP 8]: https://www.python.org/dev/peps/pep-0008/
+
+
+## Running tests
+
+There is a [`./run_tests`](./run_tests) helper script for quickly invoking all
+of our unittests.  The coverage isn't great currently, but it should still be
+run for all commits.
+
+Adding more unittests for changes you make would be greatly appreciated :).
+Check out the [tests/](./tests/) subdirectory for more details.
+
+
+## Check the license
+
+repo is licensed under the Apache License, 2.0.
+
+Because of this licensing model *every* file within the project
+*must* list the license that covers it in the header of the file.
+Any new contributions to an existing file *must* be submitted under
+the current license of that file.  Any new files *must* clearly
+indicate which license they are provided under in the file header.
+
+Please verify that you are legally allowed and willing to submit your
+changes under the license covering each file *prior* to submitting
+your patch.  It is virtually impossible to remove a patch once it
+has been applied and pushed out.
+
+
+## Sending your patches.
+
+Do not email your patches to anyone.
+
+Instead, login to the Gerrit Code Review tool at:
+
+  https://gerrit-review.googlesource.com/
+
+Ensure you have completed one of the necessary contributor
+agreements, providing documentation to the project maintainers that
+they have right to redistribute your work under the Apache License:
+
+  https://gerrit-review.googlesource.com/#/settings/agreements
+
+Ensure you have obtained an HTTP password to authenticate:
+
+  https://gerrit-review.googlesource.com/new-password
+
+Ensure that you have the local commit hook installed to automatically
+add a ChangeId to your commits:
+
+    curl -Lo `git rev-parse --git-dir`/hooks/commit-msg https://gerrit-review.googlesource.com/tools/hooks/commit-msg
+    chmod +x `git rev-parse --git-dir`/hooks/commit-msg
+
+If you have already committed your changes you will need to amend the commit
+to get the ChangeId added.
+
+    git commit --amend
+
+Push your patches over HTTPS to the review server, possibly through
+a remembered remote to make this easier in the future:
+
+    git config remote.review.url https://gerrit-review.googlesource.com/git-repo
+    git config remote.review.push HEAD:refs/for/master
+
+    git push review
+
+You will be automatically emailed a copy of your commits, and any
+comments made by the project maintainers.
+
+
+## Make changes if requested
+
+The project maintainer who reviews your changes might request changes to your
+commit. If you make the requested changes you will need to amend your commit
+and push it to the review server again.
+
+
+## Verify your changes on gerrit
+
+After you receive a Code-Review+2 from the maintainer, select the Verified
+button on the gerrit page for the change. This verifies that you have tested
+your changes and notifies the maintainer that they are ready to be submitted.
+The maintainer will then submit your changes to the repository.

repo/color.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+import pager
+
+COLORS = {None: -1,
+          'normal': -1,
+          'black': 0,
+          'red': 1,
+          'green': 2,
+          'yellow': 3,
+          'blue': 4,
+          'magenta': 5,
+          'cyan': 6,
+          'white': 7}
+
+ATTRS = {None: -1,
+         'bold': 1,
+         'dim': 2,
+         'ul': 4,
+         'blink': 5,
+         'reverse': 7}
+
+RESET = "\033[m"
+
+
+def is_color(s):
+  return s in COLORS
+
+
+def is_attr(s):
+  return s in ATTRS
+
+
+def _Color(fg=None, bg=None, attr=None):
+  fg = COLORS[fg]
+  bg = COLORS[bg]
+  attr = ATTRS[attr]
+
+  if attr >= 0 or fg >= 0 or bg >= 0:
+    need_sep = False
+    code = "\033["
+
+    if attr >= 0:
+      code += chr(ord('0') + attr)
+      need_sep = True
+
+    if fg >= 0:
+      if need_sep:
+        code += ';'
+      need_sep = True
+
+      if fg < 8:
+        code += '3%c' % (ord('0') + fg)
+      else:
+        code += '38;5;%d' % fg
+
+    if bg >= 0:
+      if need_sep:
+        code += ';'
+
+      if bg < 8:
+        code += '4%c' % (ord('0') + bg)
+      else:
+        code += '48;5;%d' % bg
+    code += 'm'
+  else:
+    code = ''
+  return code
+
+DEFAULT = None
+
+
+def SetDefaultColoring(state):
+  """Set coloring behavior to |state|.
+
+  This is useful for overriding config options via the command line.
+  """
+  if state is None:
+    # Leave it alone -- return quick!
+    return
+
+  global DEFAULT
+  state = state.lower()
+  if state in ('auto',):
+    DEFAULT = state
+  elif state in ('always', 'yes', 'true', True):
+    DEFAULT = 'always'
+  elif state in ('never', 'no', 'false', False):
+    DEFAULT = 'never'
+
+
+class Coloring(object):
+  def __init__(self, config, section_type):
+    self._section = 'color.%s' % section_type
+    self._config = config
+    self._out = sys.stdout
+
+    on = DEFAULT
+    if on is None:
+      on = self._config.GetString(self._section)
+      if on is None:
+        on = self._config.GetString('color.ui')
+
+    if on == 'auto':
+      if pager.active or os.isatty(1):
+        self._on = True
+      else:
+        self._on = False
+    elif on in ('true', 'always'):
+      self._on = True
+    else:
+      self._on = False
+
+  def redirect(self, out):
+    self._out = out
+
+  @property
+  def is_on(self):
+    return self._on
+
+  def write(self, fmt, *args):
+    self._out.write(fmt % args)
+
+  def flush(self):
+    self._out.flush()
+
+  def nl(self):
+    self._out.write('\n')
+
+  def printer(self, opt=None, fg=None, bg=None, attr=None):
+    s = self
+    c = self.colorer(opt, fg, bg, attr)
+
+    def f(fmt, *args):
+      s._out.write(c(fmt, *args))
+    return f
+
+  def nofmt_printer(self, opt=None, fg=None, bg=None, attr=None):
+    s = self
+    c = self.nofmt_colorer(opt, fg, bg, attr)
+
+    def f(fmt):
+      s._out.write(c(fmt))
+    return f
+
+  def colorer(self, opt=None, fg=None, bg=None, attr=None):
+    if self._on:
+      c = self._parse(opt, fg, bg, attr)
+
+      def f(fmt, *args):
+        output = fmt % args
+        return ''.join([c, output, RESET])
+      return f
+    else:
+
+      def f(fmt, *args):
+        return fmt % args
+      return f
+
+  def nofmt_colorer(self, opt=None, fg=None, bg=None, attr=None):
+    if self._on:
+      c = self._parse(opt, fg, bg, attr)
+
+      def f(fmt):
+        return ''.join([c, fmt, RESET])
+      return f
+    else:
+      def f(fmt):
+        return fmt
+      return f
+
+  def _parse(self, opt, fg, bg, attr):
+    if not opt:
+      return _Color(fg, bg, attr)
+
+    v = self._config.GetString('%s.%s' % (self._section, opt))
+    if v is None:
+      return _Color(fg, bg, attr)
+
+    v = v.strip().lower()
+    if v == "reset":
+      return RESET
+    elif v == '':
+      return _Color(fg, bg, attr)
+
+    have_fg = False
+    for a in v.split(' '):
+      if is_color(a):
+        if have_fg:
+          bg = a
+        else:
+          fg = a
+      elif is_attr(a):
+        attr = a
+
+    return _Color(fg, bg, attr)

repo/command.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import optparse
+import platform
+import re
+import sys
+
+from event_log import EventLog
+from error import NoSuchProjectError
+from error import InvalidProjectGroupsError
+
+
+class Command(object):
+  """Base class for any command line action in repo.
+  """
+
+  common = False
+  event_log = EventLog()
+  manifest = None
+  _optparse = None
+
+  def WantPager(self, _opt):
+    return False
+
+  def ReadEnvironmentOptions(self, opts):
+    """ Set options from environment variables. """
+
+    env_options = self._RegisteredEnvironmentOptions()
+
+    for env_key, opt_key in env_options.items():
+      # Get the user-set option value if any
+      opt_value = getattr(opts, opt_key)
+
+      # If the value is set, it means the user has passed it as a command
+      # line option, and we should use that.  Otherwise we can try to set it
+      # with the value from the corresponding environment variable.
+      if opt_value is not None:
+        continue
+
+      env_value = os.environ.get(env_key)
+      if env_value is not None:
+        setattr(opts, opt_key, env_value)
+
+    return opts
+
+  @property
+  def OptionParser(self):
+    if self._optparse is None:
+      try:
+        me = 'repo %s' % self.NAME
+        usage = self.helpUsage.strip().replace('%prog', me)
+      except AttributeError:
+        usage = 'repo %s' % self.NAME
+      self._optparse = optparse.OptionParser(usage=usage)
+      self._Options(self._optparse)
+    return self._optparse
+
+  def _Options(self, p):
+    """Initialize the option parser.
+    """
+
+  def _RegisteredEnvironmentOptions(self):
+    """Get options that can be set from environment variables.
+
+    Return a dictionary mapping environment variable name
+    to option key name that it can override.
+
+    Example: {'REPO_MY_OPTION': 'my_option'}
+
+    Will allow the option with key value 'my_option' to be set
+    from the value in the environment variable named 'REPO_MY_OPTION'.
+
+    Note: This does not work properly for options that are explicitly
+    set to None by the user, or options that are defined with a
+    default value other than None.
+
+    """
+    return {}
+
+  def Usage(self):
+    """Display usage and terminate.
+    """
+    self.OptionParser.print_usage()
+    sys.exit(1)
+
+  def ValidateOptions(self, opt, args):
+    """Validate the user options & arguments before executing.
+
+    This is meant to help break the code up into logical steps.  Some tips:
+    * Use self.OptionParser.error to display CLI related errors.
+    * Adjust opt member defaults as makes sense.
+    * Adjust the args list, but do so inplace so the caller sees updates.
+    * Try to avoid updating self state.  Leave that to Execute.
+    """
+
+  def Execute(self, opt, args):
+    """Perform the action, after option parsing is complete.
+    """
+    raise NotImplementedError
+
+  def _ResetPathToProjectMap(self, projects):
+    self._by_path = dict((p.worktree, p) for p in projects)
+
+  def _UpdatePathToProjectMap(self, project):
+    self._by_path[project.worktree] = project
+
+  def _GetProjectByPath(self, manifest, path):
+    project = None
+    if os.path.exists(path):
+      oldpath = None
+      while path and \
+            path != oldpath and \
+            path != manifest.topdir:
+        try:
+          project = self._by_path[path]
+          break
+        except KeyError:
+          oldpath = path
+          path = os.path.dirname(path)
+      if not project and path == manifest.topdir:
+        try:
+          project = self._by_path[path]
+        except KeyError:
+          pass
+    else:
+      try:
+        project = self._by_path[path]
+      except KeyError:
+        pass
+    return project
+
+  def GetProjects(self, args, manifest=None, groups='', missing_ok=False,
+                  submodules_ok=False):
+    """A list of projects that match the arguments.
+    """
+    if not manifest:
+      manifest = self.manifest
+    all_projects_list = manifest.projects
+    result = []
+
+    mp = manifest.manifestProject
+
+    if not groups:
+      groups = mp.config.GetString('manifest.groups')
+    if not groups:
+      groups = 'default,platform-' + platform.system().lower()
+    groups = [x for x in re.split(r'[,\s]+', groups) if x]
+
+    if not args:
+      derived_projects = {}
+      for project in all_projects_list:
+        if submodules_ok or project.sync_s:
+          derived_projects.update((p.name, p)
+                                  for p in project.GetDerivedSubprojects())
+      all_projects_list.extend(derived_projects.values())
+      for project in all_projects_list:
+        if (missing_ok or project.Exists) and project.MatchesGroups(groups):
+          result.append(project)
+    else:
+      self._ResetPathToProjectMap(all_projects_list)
+
+      for arg in args:
+        # We have to filter by manifest groups in case the requested project is
+        # checked out multiple times or differently based on them.
+        projects = [project for project in manifest.GetProjectsWithName(arg)
+                    if project.MatchesGroups(groups)]
+
+        if not projects:
+          path = os.path.abspath(arg).replace('\\', '/')
+          project = self._GetProjectByPath(manifest, path)
+
+          # If it's not a derived project, update path->project mapping and
+          # search again, as arg might actually point to a derived subproject.
+          if (project and not project.Derived and (submodules_ok or
+                                                   project.sync_s)):
+            search_again = False
+            for subproject in project.GetDerivedSubprojects():
+              self._UpdatePathToProjectMap(subproject)
+              search_again = True
+            if search_again:
+              project = self._GetProjectByPath(manifest, path) or project
+
+          if project:
+            projects = [project]
+
+        if not projects:
+          raise NoSuchProjectError(arg)
+
+        for project in projects:
+          if not missing_ok and not project.Exists:
+            raise NoSuchProjectError('%s (%s)' % (arg, project.relpath))
+          if not project.MatchesGroups(groups):
+            raise InvalidProjectGroupsError(arg)
+
+        result.extend(projects)
+
+    def _getpath(x):
+      return x.relpath
+    result.sort(key=_getpath)
+    return result
+
+  def FindProjects(self, args, inverse=False):
+    result = []
+    patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args]
+    for project in self.GetProjects(''):
+      for pattern in patterns:
+        match = pattern.search(project.name) or pattern.search(project.relpath)
+        if not inverse and match:
+          result.append(project)
+          break
+        if inverse and match:
+          break
+      else:
+        if inverse:
+          result.append(project)
+    result.sort(key=lambda project: project.relpath)
+    return result
+
+
+class InteractiveCommand(Command):
+  """Command which requires user interaction on the tty and
+     must not run within a pager, even if the user asks to.
+  """
+  def WantPager(self, _opt):
+    return False
+
+
+class PagedCommand(Command):
+  """Command which defaults to output in a pager, as its
+     display tends to be larger than one screen full.
+  """
+  def WantPager(self, _opt):
+    return True
+
+
+class MirrorSafeCommand(object):
+  """Command permits itself to run within a mirror,
+     and does not require a working directory.
+  """
+
+
+class GitcAvailableCommand(object):
+  """Command that requires GITC to be available, but does
+     not require the local client to be a GITC client.
+  """
+
+
+class GitcClientCommand(object):
+  """Command that requires the local client to be a GITC
+     client.
+  """

repo/docs/python-support.md

+# Supported Python Versions
+
+With Python 2.7 officially going EOL on [01 Jan 2020](https://pythonclock.org/),
+we need a support plan for the repo project itself.
+Inevitably, there will be a long tail of users who still want to use Python 2 on
+their old LTS/corp systems and have little power to change the system.
+
+## Summary
+
+* Python 3.6 (released Dec 2016) is required by default starting with repo-1.14.
+* Older versions of Python (e.g. v2.7) may use the legacy feature-frozen branch
+  based on repo-1.13.
+
+## Overview
+
+We provide a branch for Python 2 users that is feature-frozen.
+Bugfixes may be added on a best-effort basis or from the community, but largely
+no new features will be added, nor is support guaranteed.
+
+Users can select this during `repo init` time via the [repo launcher].
+Otherwise the default branches (e.g. stable & master) will be used which will
+require Python 3.
+
+This means the [repo launcher] needs to support both Python 2 & Python 3, but
+since it doesn't import any other repo code, this shouldn't be too problematic.
+
+The master branch will require Python 3.6 at a minimum.
+If the system has an older version of Python 3, then users will have to select
+the legacy Python 2 branch instead.
+
+### repo hooks
+
+Projects that use [repo hooks] run on independent schedules.
+They might migrate to Python 3 earlier or later than us.
+To support them, we'll probe the shebang of the hook script and if we find an
+interpreter in there that indicates a different version than repo is currently
+running under, we'll attempt to reexec ourselves under that.
+
+For example, a hook with a header like `#!/usr/bin/python2` will have repo
+execute `/usr/bin/python2` to execute the hook code specifically if repo is
+currently running Python 3.
+
+For more details, consult the [repo hooks] documentation.
+
+
+[repo hooks]: ./repo-hooks.md
+[repo launcher]: ../repo

repo/docs/release-process.md

+# repo release process
+
+This is the process for creating a new release of repo, as well as all the
+related topics and flows.
+
+[TOC]
+
+## Launcher script
+
+The main repo script serves as a standalone program and is often referred to as
+the "launcher script".
+This makes it easy to copy around and install as you don't have to install any
+other files from the git repo.
+
+Whenever major changes are made to the launcher script, you should increment the
+`VERSION` variable in the launcher itself.
+At runtime, repo will check this to see if it needs to be updated (and notify
+the user automatically).
+
+## Key management
+
+Every release has a git tag that is signed with a key that repo recognizes.
+Those keys are hardcoded inside of the repo launcher itself -- look for the
+`KEYRING_VERSION` and `MAINTAINER_KEYS` settings.
+
+Adding new keys to the repo launcher will allow tags to be recognized by new
+keys, but only people using that updated version will be able to.
+Since the majority of users will be using an official launcher version, their
+version will simply ignore any new signed tags.
+
+If you want to add new keys, it's best to register them long ahead of time,
+and then wait for that updated launcher to make its way out to everyone.
+Even then, there will be a long tail of users with outdated launchers, so be
+prepared for people asking questions.
+
+### Registering a new key
+
+The process of actually adding a new key is quite simple.
+
+1.  Add the public half of the key to `MAINTAINER_KEYS`.
+2.  Increment `KEYRING_VERSION` so repo knows it needs to update.
+3.  Wait a long time after that version is in a release (~months) before trying
+    to create a new release using those new keys.
+
+## Self update algorithm
+
+When creating a new repo checkout with `repo init`, there are a few options that
+control how repo finds updates:
+
+*   `--repo-url`: This tells repo where to clone the full repo project itself.
+    It defaults to the official project (`REPO_URL` in the launcher script).
+*   `--repo-branch`: This tells repo which branch to use for the full project.
+    It defaults to the `stable` branch (`REPO_REV` in the launcher script).
+
+Whenever `repo sync` is run, repo will check to see if an update is available.
+It fetches the latest repo-branch from the repo-url.
+Then it verifies that the latest commit in the branch has a valid signed tag
+using `git tag -v` (which uses gpg).
+If the tag is valid, then repo will update its internal checkout to it.
+
+If the latest commit doesn't have a signed tag, repo will fall back to the
+most recent tag it can find (via `git describe`).
+If that tag is valid, then repo will warn and use that commit instead.
+
+If that tag cannot be verified, it gives up and forces the user to resolve.
+
+## Branch management
+
+All development happens on the `master` branch and should generally be stable.
+
+Since the repo launcher defaults to tracking the `stable` branch, it is not
+normally updated until a new release is available.
+If something goes wrong with a new release, an older release can be force pushed
+and clients will automatically downgrade.
+
+The `maint` branch is used to track the previous major release of repo.
+It is not normally meant to be used by people as `stable` should be good enough.
+Once a new major release is pushed to the `stable` branch, then the previous
+major release can be pushed to `maint`.
+For example, when `stable` moves from `v1.10.x` to `v1.11.x`, then the `maint`
+branch will be updated from `v1.9.x` to `v1.10.x`.
+
+We don't have parallel release branches/series.
+Typically all tags are made against the `master` branch and then pushed to the
+`stable` branch to make it available to the rest of the world.
+Since repo doesn't typically see a lot of changes, this tends to be OK.
+
+## Creating a new release
+
+When you want to create a new release, you'll need to select a good version and
+create a signed tag using a key registered in repo itself.
+Typically we just tag the latest version of the `master` branch.
+The tag could be pushed now, but it won't be used by clients normally (since the
+default `repo-branch` setting is `stable`).
+This would allow some early testing on systems who explicitly select `master`.
+
+### Creating a signed tag
+
+Lets assume your keys live in a dedicated directory, e.g. `~/.gnupg/repo/`.
+
+*** note
+If you need access to the official keys, check out the internal documentation
+at [go/repo-release].
+Note that only official maintainers of repo will have access as it describes
+internal processes for accessing the restricted keys.
+***
+
+```sh
+# Set the gpg key directory.
+$ export GNUPGHOME=~/.gnupg/repo/
+
+# Verify the listed key is “Repo Maintainer”.
+$ gpg -K
+
+# Pick whatever branch or commit you want to tag.
+$ r=master
+
+# Pick the new version.
+$ t=1.12.10
+
+# Create the signed tag.
+$ git tag -s v$t -u "Repo Maintainer <repo@android.kernel.org>" -m "repo $t" $r
+
+# Verify the signed tag.
+$ git show v$t
+```
+
+### Push the new release
+
+Once you're ready to make the release available to everyone, push it to the
+`stable` branch.
+
+Make sure you never push the tag itself to the stable branch!
+Only push the commit -- notice the use of `$t` and `$r` below.
+
+```sh
+$ git push https://gerrit-review.googlesource.com/git-repo v$t
+$ git push https://gerrit-review.googlesource.com/git-repo $r:stable
+```
+
+If something goes horribly wrong, you can force push the previous version to the
+`stable` branch and people should automatically recover.
+Again, make sure you never push the tag itself!
+
+```sh
+$ oldrev="whatever-old-commit"
+$ git push https://gerrit-review.googlesource.com/git-repo $oldrev:stable --force
+```
+
+### Announce the release
+
+Once you do push a new release to `stable`, make sure to announce it on the
+[repo-discuss@googlegroups.com] group.
+Here is an [example announcement].
+
+You can create a short changelog using the command:
+
+```sh
+# If you haven't pushed to the stable branch yet, you can use origin/stable.
+# If you have pushed, change origin/stable to the previous release tag.
+$ git log --format="%h (%aN) %s" --no-merges origin/stable..$r
+```
+
+
+[example announcement]: https://groups.google.com/d/topic/repo-discuss/UGBNismWo1M/discussion
+[repo-discuss@googlegroups.com]: https://groups.google.com/forum/#!forum/repo-discuss
+[go/repo-release]: https://goto.google.com/repo-release

repo/docs/repo-hooks.md

+# repo hooks
+
+[TOC]
+
+Repo provides a mechanism to hook specific stages of the runtime with custom
+python modules.  All the hooks live in one git project which is checked out by
+the manifest (specified during `repo init`), and the manifest itself defines
+which hooks are registered.
+
+These are useful to run linters, check formatting, and run quick unittests
+before allowing a step to proceed (e.g. before uploading a commit to Gerrit).
+
+A complete example can be found in the Android project.  It can be easily
+re-used by any repo based project and is not specific to Android.<br>
+https://android.googlesource.com/platform/tools/repohooks
+
+## Approvals
+
+When a hook is processed the first time, the user is prompted for approval.
+We don't want to execute arbitrary code without explicit consent.  For manifests
+fetched via secure protocols (e.g. https://), the user is prompted once.  For
+insecure protocols (e.g. http://), the user is prompted whenever the registered
+repohooks project is updated and a hook is triggered.
+
+## Manifest Settings
+
+For the full syntax, see the [repo manifest format](./manifest-format.md).
+
+Here's a short example from
+[Android](https://android.googlesource.com/platform/manifest/+/master/default.xml).
+The `<project>` line checks out the repohooks git repo to the local
+`tools/repohooks/` path.  The `<repo-hooks>` line says to look in the project
+with the name `platform/tools/repohooks` for hooks to run during the
+`pre-upload` phase.
+
+```xml
+<project path="tools/repohooks" name="platform/tools/repohooks" />
+<repo-hooks in-project="platform/tools/repohooks" enabled-list="pre-upload" />
+```
+
+## Source Layout
+
+The repohooks git repo should have a python file with the same name as the hook.
+So if you want to support the `pre-upload` hook, you'll need to create a file
+named `pre-upload.py`.  Repo will dynamically load that module when processing
+the hook and then call the `main` function in it.
+
+Hooks should have their `main` accept `**kwargs` for future compatibility.
+
+## Runtime
+
+Hook return values are ignored.
+
+Any uncaught exceptions from the hook will cause the step to fail.  This is
+intended as a fallback safety check though rather than the normal flow.  If
+you want your hook to trigger a failure, it should call `sys.exit()` (after
+displaying relevant diagnostics).
+
+Output (stdout & stderr) are not filtered in any way.  Hooks should generally
+not be too verbose.  A short summary is nice, and some status information when
+long running operations occur, but long/verbose output should be used only if
+the hook ultimately fails.
+
+The hook runs from the top level of the repo client where the operation is
+started.
+For example, if the repo client is under `~/tree/`, then that is where the hook
+runs, even if you ran repo in a git repository at `~/tree/src/foo/`, or in a
+subdirectory of that git repository in `~/tree/src/foo/bar/`.
+Hooks frequently start off by doing a `os.chdir` to the specific project they're
+called on (see below) and then changing back to the original dir when they're
+finished.
+
+Python's `sys.path` is modified so that the top of repohooks directory comes
+first.  This should help simplify the hook logic to easily allow importing of
+local modules.
+
+Repo does not modify the state of the git checkout.  This means that the hooks
+might be running in a dirty git repo with many commits and checked out to the
+latest one.  If the hook wants to operate on specific git commits, it needs to
+manually discover the list of pending commits, extract the diff/commit, and
+then check it directly.  Hooks should not normally modify the active git repo
+(such as checking out a specific commit to run checks) without first prompting
+the user.  Although user interaction is discouraged in the common case, it can
+be useful when deploying automatic fixes.
+
+### Shebang Handling
+
+*** note
+This is intended as a transitional feature.  Hooks are expected to eventually
+migrate to Python 3 only as Python 2 is EOL & deprecated.
+***
+
+If the hook is written against a specific version of Python (either 2 or 3),
+the script can declare that explicitly.  Repo will then attempt to execute it
+under the right version of Python regardless of the version repo itself might
+be executing under.
+
+Here are the shebangs that are recognized.
+
+* `#!/usr/bin/env python` & `#!/usr/bin/python`: The hook is compatible with
+  Python 2 & Python 3.  For maximum compatibility, these are recommended.
+* `#!/usr/bin/env python2` & `#!/usr/bin/python2`: The hook requires Python 2.
+  Version specific names like `python2.7` are also recognized.
+* `#!/usr/bin/env python3` & `#!/usr/bin/python3`: The hook requires Python 3.
+  Version specific names like `python3.6` are also recognized.
+
+If no shebang is detected, or does not match the forms above, we assume that the
+hook is compatible with both Python 2 & Python 3 as if `#!/usr/bin/python` was
+used.
+
+## Hooks
+
+Here are all the points available for hooking.
+
+### pre-upload
+
+This hook runs when people run `repo upload`.
+
+The `pre-upload.py` file should be defined like:
+
+```py
+def main(project_list, worktree_list=None, **kwargs):
+    """Main function invoked directly by repo.
+
+    We must use the name "main" as that is what repo requires.
+
+    Args:
+      project_list: List of projects to run on.
+      worktree_list: A list of directories.  It should be the same length as
+          project_list, so that each entry in project_list matches with a
+          directory in worktree_list.  If None, we will attempt to calculate
+          the directories automatically.
+      kwargs: Leave this here for forward-compatibility.
+    """
+```

repo/editor.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import re
+import sys
+import subprocess
+import tempfile
+
+from error import EditorError
+import platform_utils
+
+class Editor(object):
+  """Manages the user's preferred text editor."""
+
+  _editor = None
+  globalConfig = None
+
+  @classmethod
+  def _GetEditor(cls):
+    if cls._editor is None:
+      cls._editor = cls._SelectEditor()
+    return cls._editor
+
+  @classmethod
+  def _SelectEditor(cls):
+    e = os.getenv('GIT_EDITOR')
+    if e:
+      return e
+
+    if cls.globalConfig:
+      e = cls.globalConfig.GetString('core.editor')
+      if e:
+        return e
+
+    e = os.getenv('VISUAL')
+    if e:
+      return e
+
+    e = os.getenv('EDITOR')
+    if e:
+      return e
+
+    if os.getenv('TERM') == 'dumb':
+      print(
+"""No editor specified in GIT_EDITOR, core.editor, VISUAL or EDITOR.
+Tried to fall back to vi but terminal is dumb.  Please configure at
+least one of these before using this command.""", file=sys.stderr)
+      sys.exit(1)
+
+    return 'vi'
+
+  @classmethod
+  def EditString(cls, data):
+    """Opens an editor to edit the given content.
+
+       Args:
+         data        : the text to edit
+
+      Returns:
+        new value of edited text; None if editing did not succeed
+    """
+    editor = cls._GetEditor()
+    if editor == ':':
+      return data
+
+    fd, path = tempfile.mkstemp()
+    try:
+      os.write(fd, data)
+      os.close(fd)
+      fd = None
+
+      if platform_utils.isWindows():
+        # Split on spaces, respecting quoted strings
+        import shlex
+        args = shlex.split(editor)
+        shell = False
+      elif re.compile("^.*[$ \t'].*$").match(editor):
+        args = [editor + ' "$@"', 'sh']
+        shell = True
+      else:
+        args = [editor]
+        shell = False
+      args.append(path)
+
+      try:
+        rc = subprocess.Popen(args, shell=shell).wait()
+      except OSError as e:
+        raise EditorError('editor failed, %s: %s %s'
+          % (str(e), editor, path))
+      if rc != 0:
+        raise EditorError('editor failed with exit status %d: %s %s'
+          % (rc, editor, path))
+
+      fd2 = open(path)
+      try:
+        return fd2.read()
+      finally:
+        fd2.close()
+    finally:
+      if fd:
+        os.close(fd)
+      platform_utils.remove(path)

repo/error.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class ManifestParseError(Exception):
+  """Failed to parse the manifest file.
+  """
+
+class ManifestInvalidRevisionError(Exception):
+  """The revision value in a project is incorrect.
+  """
+
+class NoManifestException(Exception):
+  """The required manifest does not exist.
+  """
+  def __init__(self, path, reason):
+    super(NoManifestException, self).__init__()
+    self.path = path
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
+class EditorError(Exception):
+  """Unspecified error from the user's text editor.
+  """
+  def __init__(self, reason):
+    super(EditorError, self).__init__()
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
+class GitError(Exception):
+  """Unspecified internal error from git.
+  """
+  def __init__(self, command):
+    super(GitError, self).__init__()
+    self.command = command
+
+  def __str__(self):
+    return self.command
+
+class UploadError(Exception):
+  """A bundle upload to Gerrit did not succeed.
+  """
+  def __init__(self, reason):
+    super(UploadError, self).__init__()
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
+class DownloadError(Exception):
+  """Cannot download a repository.
+  """
+  def __init__(self, reason):
+    super(DownloadError, self).__init__()
+    self.reason = reason
+
+  def __str__(self):
+    return self.reason
+
+class NoSuchProjectError(Exception):
+  """A specified project does not exist in the work tree.
+  """
+  def __init__(self, name=None):
+    super(NoSuchProjectError, self).__init__()
+    self.name = name
+
+  def __str__(self):
+    if self.name is None:
+      return 'in current directory'
+    return self.name
+
+
+class InvalidProjectGroupsError(Exception):
+  """A specified project is not suitable for the specified groups
+  """
+  def __init__(self, name=None):
+    super(InvalidProjectGroupsError, self).__init__()
+    self.name = name
+
+  def __str__(self):
+    if self.name is None:
+      return 'in current directory'
+    return self.name
+
+class RepoChangedException(Exception):
+  """Thrown if 'repo sync' results in repo updating its internal
+     repo or manifest repositories.  In this special case we must
+     use exec to re-execute repo with the new code and manifest.
+  """
+  def __init__(self, extra_args=None):
+    super(RepoChangedException, self).__init__()
+    self.extra_args = extra_args or []
+
+class HookError(Exception):
+  """Thrown if a 'repo-hook' could not be run.
+
+  The common case is that the file wasn't present when we tried to run it.
+  """

repo/event_log.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2017 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+
+import json
+import multiprocessing
+
+TASK_COMMAND = 'command'
+TASK_SYNC_NETWORK = 'sync-network'
+TASK_SYNC_LOCAL = 'sync-local'
+
+class EventLog(object):
+  """Event log that records events that occurred during a repo invocation.
+
+  Events are written to the log as a consecutive JSON entries, one per line.
+  Each entry contains the following keys:
+  - id: A ('RepoOp', ID) tuple, suitable for storing in a datastore.
+        The ID is only unique for the invocation of the repo command.
+  - name: Name of the object being operated upon.
+  - task_name: The task that was performed.
+  - start: Timestamp of when the operation started.
+  - finish: Timestamp of when the operation finished.
+  - success: Boolean indicating if the operation was successful.
+  - try_count: A counter indicating the try count of this task.
+
+  Optionally:
+  - parent: A ('RepoOp', ID) tuple indicating the parent event for nested
+            events.
+
+  Valid task_names include:
+  - command: The invocation of a subcommand.
+  - sync-network: The network component of a sync command.
+  - sync-local: The local component of a sync command.
+
+  Specific tasks may include additional informational properties.
+  """
+
+  def __init__(self):
+    """Initializes the event log."""
+    self._log = []
+    self._parent = None
+
+  def Add(self, name, task_name, start, finish=None, success=None,
+          try_count=1, kind='RepoOp'):
+    """Add an event to the log.
+
+    Args:
+      name: Name of the object being operated upon.
+      task_name: A sub-task that was performed for name.
+      start: Timestamp of when the operation started.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+      try_count: A counter indicating the try count of this task.
+      kind: The kind of the object for the unique identifier.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event = {
+        'id': (kind, _NextEventId()),
+        'name': name,
+        'task_name': task_name,
+        'start_time': start,
+        'try': try_count,
+    }
+
+    if self._parent:
+      event['parent'] = self._parent['id']
+
+    if success is not None or finish is not None:
+        self.FinishEvent(event, finish, success)
+
+    self._log.append(event)
+    return event
+
+  def AddSync(self, project, task_name, start, finish, success):
+    """Add a event to the log for a sync command.
+
+    Args:
+      project: Project being synced.
+      task_name: A sub-task that was performed for name.
+                 One of (TASK_SYNC_NETWORK, TASK_SYNC_LOCAL)
+      start: Timestamp of when the operation started.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event = self.Add(project.relpath, task_name, start, finish, success)
+    if event is not None:
+      event['project'] = project.name
+      if project.revisionExpr:
+        event['revision'] = project.revisionExpr
+      if project.remote.url:
+        event['project_url'] = project.remote.url
+      if project.remote.fetchUrl:
+        event['remote_url'] = project.remote.fetchUrl
+      try:
+        event['git_hash'] = project.GetCommitRevisionId()
+      except Exception:
+        pass
+    return event
+
+  def GetStatusString(self, success):
+    """Converst a boolean success to a status string.
+
+    Args:
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      status string.
+    """
+    return 'pass' if success else 'fail'
+
+  def FinishEvent(self, event, finish, success):
+    """Finishes an incomplete event.
+
+    Args:
+      event: An event that has been added to the log.
+      finish: Timestamp of when the operation finished.
+      success: Boolean indicating if the operation was successful.
+
+    Returns:
+      A dictionary of the event added to the log.
+    """
+    event['status'] =  self.GetStatusString(success)
+    event['finish_time'] = finish
+    return event
+
+  def SetParent(self, event):
+    """Set a parent event for all new entities.
+
+    Args:
+      event: The event to use as a parent.
+    """
+    self._parent = event
+
+  def Write(self, filename):
+    """Writes the log out to a file.
+
+    Args:
+      filename: The file to write the log to.
+    """
+    with open(filename, 'w+') as f:
+      for e in self._log:
+        json.dump(e, f, sort_keys=True)
+        f.write('\n')
+
+
+# An integer id that is unique across this invocation of the program.
+_EVENT_ID = multiprocessing.Value('i', 1)
+
+def _NextEventId():
+  """Helper function for grabbing the next unique id.
+
+  Returns:
+    A unique, to this invocation of the program, integer id.
+  """
+  with _EVENT_ID.get_lock():
+    val = _EVENT_ID.value
+    _EVENT_ID.value += 1
+  return val

repo/git_command.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import tempfile
+from signal import SIGTERM
+
+from error import GitError
+from git_refs import HEAD
+import platform_utils
+from repo_trace import REPO_TRACE, IsTrace, Trace
+from wrapper import Wrapper
+
+GIT = 'git'
+MIN_GIT_VERSION = (1, 5, 4)
+GIT_DIR = 'GIT_DIR'
+
+LAST_GITDIR = None
+LAST_CWD = None
+
+_ssh_proxy_path = None
+_ssh_sock_path = None
+_ssh_clients = []
+
+def ssh_sock(create=True):
+  global _ssh_sock_path
+  if _ssh_sock_path is None:
+    if not create:
+      return None
+    tmp_dir = '/tmp'
+    if not os.path.exists(tmp_dir):
+      tmp_dir = tempfile.gettempdir()
+    _ssh_sock_path = os.path.join(
+      tempfile.mkdtemp('', 'ssh-', tmp_dir),
+      'master-%r@%h:%p')
+  return _ssh_sock_path
+
+def _ssh_proxy():
+  global _ssh_proxy_path
+  if _ssh_proxy_path is None:
+    _ssh_proxy_path = os.path.join(
+      os.path.dirname(__file__),
+      'git_ssh')
+  return _ssh_proxy_path
+
+def _add_ssh_client(p):
+  _ssh_clients.append(p)
+
+def _remove_ssh_client(p):
+  try:
+    _ssh_clients.remove(p)
+  except ValueError:
+    pass
+
+def terminate_ssh_clients():
+  global _ssh_clients
+  for p in _ssh_clients:
+    try:
+      os.kill(p.pid, SIGTERM)
+      p.wait()
+    except OSError:
+      pass
+  _ssh_clients = []
+
+_git_version = None
+
+class _GitCall(object):
+  def version_tuple(self):
+    global _git_version
+    if _git_version is None:
+      _git_version = Wrapper().ParseGitVersion()
+      if _git_version is None:
+        print('fatal: unable to detect git version', file=sys.stderr)
+        sys.exit(1)
+    return _git_version
+
+  def __getattr__(self, name):
+    name = name.replace('_','-')
+    def fun(*cmdv):
+      command = [name]
+      command.extend(cmdv)
+      return GitCommand(None, command).Wait() == 0
+    return fun
+git = _GitCall()
+
+
+def RepoSourceVersion():
+  """Return the version of the repo.git tree."""
+  ver = getattr(RepoSourceVersion, 'version', None)
+
+  # We avoid GitCommand so we don't run into circular deps -- GitCommand needs
+  # to initialize version info we provide.
+  if ver is None:
+    env = GitCommand._GetBasicEnv()
+
+    proj = os.path.dirname(os.path.abspath(__file__))
+    env[GIT_DIR] = os.path.join(proj, '.git')
+
+    p = subprocess.Popen([GIT, 'describe', HEAD], stdout=subprocess.PIPE,
+                         env=env)
+    if p.wait() == 0:
+      ver = p.stdout.read().strip().decode('utf-8')
+      if ver.startswith('v'):
+        ver = ver[1:]
+    else:
+      ver = 'unknown'
+    setattr(RepoSourceVersion, 'version', ver)
+
+  return ver
+
+
+class UserAgent(object):
+  """Mange User-Agent settings when talking to external services
+
+  We follow the style as documented here:
+  https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
+  """
+
+  _os = None
+  _repo_ua = None
+  _git_ua = None
+
+  @property
+  def os(self):
+    """The operating system name."""
+    if self._os is None:
+      os_name = sys.platform
+      if os_name.lower().startswith('linux'):
+        os_name = 'Linux'
+      elif os_name == 'win32':
+        os_name = 'Win32'
+      elif os_name == 'cygwin':
+        os_name = 'Cygwin'
+      elif os_name == 'darwin':
+        os_name = 'Darwin'
+      self._os = os_name
+
+    return self._os
+
+  @property
+  def repo(self):
+    """The UA when connecting directly from repo."""
+    if self._repo_ua is None:
+      py_version = sys.version_info
+      self._repo_ua = 'git-repo/%s (%s) git/%s Python/%d.%d.%d' % (
+          RepoSourceVersion(),
+          self.os,
+          git.version_tuple().full,
+          py_version.major, py_version.minor, py_version.micro)
+
+    return self._repo_ua
+
+  @property
+  def git(self):
+    """The UA when running git."""
+    if self._git_ua is None:
+      self._git_ua = 'git/%s (%s) git-repo/%s' % (
+          git.version_tuple().full,
+          self.os,
+          RepoSourceVersion())
+
+    return self._git_ua
+
+user_agent = UserAgent()
+
+def git_require(min_version, fail=False, msg=''):
+  git_version = git.version_tuple()
+  if min_version <= git_version:
+    return True
+  if fail:
+    need = '.'.join(map(str, min_version))
+    if msg:
+      msg = ' for ' + msg
+    print('fatal: git %s or later required%s' % (need, msg), file=sys.stderr)
+    sys.exit(1)
+  return False
+
+def _setenv(env, name, value):
+  env[name] = value.encode()
+
+class GitCommand(object):
+  def __init__(self,
+               project,
+               cmdv,
+               bare = False,
+               provide_stdin = False,
+               capture_stdout = False,
+               capture_stderr = False,
+               disable_editor = False,
+               ssh_proxy = False,
+               cwd = None,
+               gitdir = None):
+    env = self._GetBasicEnv()
+
+    # If we are not capturing std* then need to print it.
+    self.tee = {'stdout': not capture_stdout, 'stderr': not capture_stderr}
+
+    if disable_editor:
+      _setenv(env, 'GIT_EDITOR', ':')
+    if ssh_proxy:
+      _setenv(env, 'REPO_SSH_SOCK', ssh_sock())
+      _setenv(env, 'GIT_SSH', _ssh_proxy())
+      _setenv(env, 'GIT_SSH_VARIANT', 'ssh')
+    if 'http_proxy' in env and 'darwin' == sys.platform:
+      s = "'http.proxy=%s'" % (env['http_proxy'],)
+      p = env.get('GIT_CONFIG_PARAMETERS')
+      if p is not None:
+        s = p + ' ' + s
+      _setenv(env, 'GIT_CONFIG_PARAMETERS', s)
+    if 'GIT_ALLOW_PROTOCOL' not in env:
+      _setenv(env, 'GIT_ALLOW_PROTOCOL',
+              'file:git:http:https:ssh:persistent-http:persistent-https:sso:rpc')
+    _setenv(env, 'GIT_HTTP_USER_AGENT', user_agent.git)
+
+    if project:
+      if not cwd:
+        cwd = project.worktree
+      if not gitdir:
+        gitdir = project.gitdir
+
+    command = [GIT]
+    if bare:
+      if gitdir:
+        _setenv(env, GIT_DIR, gitdir)
+      cwd = None
+    command.append(cmdv[0])
+    # Need to use the --progress flag for fetch/clone so output will be
+    # displayed as by default git only does progress output if stderr is a TTY.
+    if sys.stderr.isatty() and cmdv[0] in ('fetch', 'clone'):
+      if '--progress' not in cmdv and '--quiet' not in cmdv:
+        command.append('--progress')
+    command.extend(cmdv[1:])
+
+    if provide_stdin:
+      stdin = subprocess.PIPE
+    else:
+      stdin = None
+
+    stdout = subprocess.PIPE
+    stderr = subprocess.PIPE
+
+    if IsTrace():
+      global LAST_CWD
+      global LAST_GITDIR
+
+      dbg = ''
+
+      if cwd and LAST_CWD != cwd:
+        if LAST_GITDIR or LAST_CWD:
+          dbg += '\n'
+        dbg += ': cd %s\n' % cwd
+        LAST_CWD = cwd
+
+      if GIT_DIR in env and LAST_GITDIR != env[GIT_DIR]:
+        if LAST_GITDIR or LAST_CWD:
+          dbg += '\n'
+        dbg += ': export GIT_DIR=%s\n' % env[GIT_DIR]
+        LAST_GITDIR = env[GIT_DIR]
+
+      dbg += ': '
+      dbg += ' '.join(command)
+      if stdin == subprocess.PIPE:
+        dbg += ' 0<|'
+      if stdout == subprocess.PIPE:
+        dbg += ' 1>|'
+      if stderr == subprocess.PIPE:
+        dbg += ' 2>|'
+      Trace('%s', dbg)
+
+    try:
+      p = subprocess.Popen(command,
+                           cwd = cwd,
+                           env = env,
+                           stdin = stdin,
+                           stdout = stdout,
+                           stderr = stderr)
+    except Exception as e:
+      raise GitError('%s: %s' % (command[1], e))
+
+    if ssh_proxy:
+      _add_ssh_client(p)
+
+    self.process = p
+    self.stdin = p.stdin
+
+  @staticmethod
+  def _GetBasicEnv():
+    """Return a basic env for running git under.
+
+    This is guaranteed to be side-effect free.
+    """
+    env = os.environ.copy()
+    for key in (REPO_TRACE,
+                GIT_DIR,
+                'GIT_ALTERNATE_OBJECT_DIRECTORIES',
+                'GIT_OBJECT_DIRECTORY',
+                'GIT_WORK_TREE',
+                'GIT_GRAFT_FILE',
+                'GIT_INDEX_FILE'):
+      env.pop(key, None)
+    return env
+
+  def Wait(self):
+    try:
+      p = self.process
+      rc = self._CaptureOutput()
+    finally:
+      _remove_ssh_client(p)
+    return rc
+
+  def _CaptureOutput(self):
+    p = self.process
+    s_in = platform_utils.FileDescriptorStreams.create()
+    s_in.add(p.stdout, sys.stdout, 'stdout')
+    s_in.add(p.stderr, sys.stderr, 'stderr')
+    self.stdout = ''
+    self.stderr = ''
+
+    while not s_in.is_done:
+      in_ready = s_in.select()
+      for s in in_ready:
+        buf = s.read()
+        if not buf:
+          s_in.remove(s)
+          continue
+        if not hasattr(buf, 'encode'):
+          buf = buf.decode()
+        if s.std_name == 'stdout':
+          self.stdout += buf
+        else:
+          self.stderr += buf
+        if self.tee[s.std_name]:
+          s.dest.write(buf)
+          s.dest.flush()
+    return p.wait()

repo/git_refs.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from repo_trace import Trace
+import platform_utils
+
+HEAD      = 'HEAD'
+R_CHANGES = 'refs/changes/'
+R_HEADS   = 'refs/heads/'
+R_TAGS    = 'refs/tags/'
+R_PUB     = 'refs/published/'
+R_M       = 'refs/remotes/m/'
+
+
+class GitRefs(object):
+  def __init__(self, gitdir):
+    self._gitdir = gitdir
+    self._phyref = None
+    self._symref = None
+    self._mtime = {}
+
+  @property
+  def all(self):
+    self._EnsureLoaded()
+    return self._phyref
+
+  def get(self, name):
+    try:
+      return self.all[name]
+    except KeyError:
+      return ''
+
+  def deleted(self, name):
+    if self._phyref is not None:
+      if name in self._phyref:
+        del self._phyref[name]
+
+      if name in self._symref:
+        del self._symref[name]
+
+      if name in self._mtime:
+        del self._mtime[name]
+
+  def symref(self, name):
+    try:
+      self._EnsureLoaded()
+      return self._symref[name]
+    except KeyError:
+      return ''
+
+  def _EnsureLoaded(self):
+    if self._phyref is None or self._NeedUpdate():
+      self._LoadAll()
+
+  def _NeedUpdate(self):
+    Trace(': scan refs %s', self._gitdir)
+
+    for name, mtime in self._mtime.items():
+      try:
+        if mtime != os.path.getmtime(os.path.join(self._gitdir, name)):
+          return True
+      except OSError:
+        return True
+    return False
+
+  def _LoadAll(self):
+    Trace(': load refs %s', self._gitdir)
+
+    self._phyref = {}
+    self._symref = {}
+    self._mtime = {}
+
+    self._ReadPackedRefs()
+    self._ReadLoose('refs/')
+    self._ReadLoose1(os.path.join(self._gitdir, HEAD), HEAD)
+
+    scan = self._symref
+    attempts = 0
+    while scan and attempts < 5:
+      scan_next = {}
+      for name, dest in scan.items():
+        if dest in self._phyref:
+          self._phyref[name] = self._phyref[dest]
+        else:
+          scan_next[name] = dest
+      scan = scan_next
+      attempts += 1
+
+  def _ReadPackedRefs(self):
+    path = os.path.join(self._gitdir, 'packed-refs')
+    try:
+      fd = open(path, 'r')
+      mtime = os.path.getmtime(path)
+    except IOError:
+      return
+    except OSError:
+      return
+    try:
+      for line in fd:
+        line = str(line)
+        if line[0] == '#':
+          continue
+        if line[0] == '^':
+          continue
+
+        line = line[:-1]
+        p = line.split(' ')
+        ref_id = p[0]
+        name = p[1]
+
+        self._phyref[name] = ref_id
+    finally:
+      fd.close()
+    self._mtime['packed-refs'] = mtime
+
+  def _ReadLoose(self, prefix):
+    base = os.path.join(self._gitdir, prefix)
+    for name in platform_utils.listdir(base):
+      p = os.path.join(base, name)
+      if platform_utils.isdir(p):
+        self._mtime[prefix] = os.path.getmtime(base)
+        self._ReadLoose(prefix + name + '/')
+      elif name.endswith('.lock'):
+        pass
+      else:
+        self._ReadLoose1(p, prefix + name)
+
+  def _ReadLoose1(self, path, name):
+    try:
+      fd = open(path)
+    except IOError:
+      return
+
+    try:
+      try:
+        mtime = os.path.getmtime(path)
+        ref_id = fd.readline()
+      except (IOError, OSError):
+        return
+    finally:
+      fd.close()
+
+    try:
+      ref_id = ref_id.decode()
+    except AttributeError:
+      pass
+    if not ref_id:
+      return
+    ref_id = ref_id[:-1]
+
+    if ref_id.startswith('ref: '):
+      self._symref[name] = ref_id[5:]
+    else:
+      self._phyref[name] = ref_id
+    self._mtime[name] = mtime

repo/git_ssh

+#!/bin/sh
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+exec ssh -o "ControlMaster no" -o "ControlPath $REPO_SSH_SOCK" "$@"

repo/gitc_utils.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import platform
+import re
+import sys
+import time
+
+import git_command
+import git_config
+import wrapper
+
+from error import ManifestParseError
+
+NUM_BATCH_RETRIEVE_REVISIONID = 32
+
+def get_gitc_manifest_dir():
+  return wrapper.Wrapper().get_gitc_manifest_dir()
+
+def parse_clientdir(gitc_fs_path):
+  return wrapper.Wrapper().gitc_parse_clientdir(gitc_fs_path)
+
+def _set_project_revisions(projects):
+  """Sets the revisionExpr for a list of projects.
+
+  Because of the limit of open file descriptors allowed, length of projects
+  should not be overly large. Recommend calling this function multiple times
+  with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+  @param projects: List of project objects to set the revionExpr for.
+  """
+  # Retrieve the commit id for each project based off of it's current
+  # revisionExpr and it is not already a commit id.
+  project_gitcmds = [(
+      project, git_command.GitCommand(None,
+                                      ['ls-remote',
+                                       project.remote.url,
+                                       project.revisionExpr],
+                                      capture_stdout=True, cwd='/tmp'))
+      for project in projects if not git_config.IsId(project.revisionExpr)]
+  for proj, gitcmd in project_gitcmds:
+    if gitcmd.Wait():
+      print('FATAL: Failed to retrieve revisionExpr for %s' % proj)
+      sys.exit(1)
+    revisionExpr = gitcmd.stdout.split('\t')[0]
+    if not revisionExpr:
+      raise ManifestParseError('Invalid SHA-1 revision project %s (%s)' %
+                               (proj.remote.url, proj.revisionExpr))
+    proj.revisionExpr = revisionExpr
+
+def _manifest_groups(manifest):
+  """Returns the manifest group string that should be synced
+
+  This is the same logic used by Command.GetProjects(), which is used during
+  repo sync
+
+  @param manifest: The XmlManifest object
+  """
+  mp = manifest.manifestProject
+  groups = mp.config.GetString('manifest.groups')
+  if not groups:
+    groups = 'default,platform-' + platform.system().lower()
+  return groups
+
+def generate_gitc_manifest(gitc_manifest, manifest, paths=None):
+  """Generate a manifest for shafsd to use for this GITC client.
+
+  @param gitc_manifest: Current gitc manifest, or None if there isn't one yet.
+  @param manifest: A GitcManifest object loaded with the current repo manifest.
+  @param paths: List of project paths we want to update.
+  """
+
+  print('Generating GITC Manifest by fetching revision SHAs for each '
+        'project.')
+  if paths is None:
+    paths = list(manifest.paths.keys())
+
+  groups = [x for x in re.split(r'[,\s]+', _manifest_groups(manifest)) if x]
+
+  # Convert the paths to projects, and filter them to the matched groups.
+  projects = [manifest.paths[p] for p in paths]
+  projects = [p for p in projects if p.MatchesGroups(groups)]
+
+  if gitc_manifest is not None:
+    for path, proj in manifest.paths.items():
+      if not proj.MatchesGroups(groups):
+        continue
+
+      if not proj.upstream and not git_config.IsId(proj.revisionExpr):
+        proj.upstream = proj.revisionExpr
+
+      if not path in gitc_manifest.paths:
+        # Any new projects need their first revision, even if we weren't asked
+        # for them.
+        projects.append(proj)
+      elif not path in paths:
+        # And copy revisions from the previous manifest if we're not updating
+        # them now.
+        gitc_proj = gitc_manifest.paths[path]
+        if gitc_proj.old_revision:
+          proj.revisionExpr = None
+          proj.old_revision = gitc_proj.old_revision
+        else:
+          proj.revisionExpr = gitc_proj.revisionExpr
+
+  index = 0
+  while index < len(projects):
+    _set_project_revisions(
+        projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+    index += NUM_BATCH_RETRIEVE_REVISIONID
+
+  if gitc_manifest is not None:
+    for path, proj in gitc_manifest.paths.items():
+      if proj.old_revision and path in paths:
+        # If we updated a project that has been started, keep the old-revision
+        # updated.
+        repo_proj = manifest.paths[path]
+        repo_proj.old_revision = repo_proj.revisionExpr
+        repo_proj.revisionExpr = None
+
+  # Convert URLs from relative to absolute.
+  for _name, remote in manifest.remotes.items():
+    remote.fetchUrl = remote.resolvedFetchUrl
+
+  # Save the manifest.
+  save_manifest(manifest)
+
+def save_manifest(manifest, client_dir=None):
+  """Save the manifest file in the client_dir.
+
+  @param client_dir: Client directory to save the manifest in.
+  @param manifest: Manifest object to save.
+  """
+  if not client_dir:
+    client_dir = manifest.gitc_client_dir
+  with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+    manifest.Save(f, groups=_manifest_groups(manifest))
+  # TODO(sbasi/jorg): Come up with a solution to remove the sleep below.
+  # Give the GITC filesystem time to register the manifest changes.
+  time.sleep(3)

repo/hooks/commit-msg

+#!/bin/sh
+# From Gerrit Code Review 2.14.6
+#
+# Part of Gerrit Code Review (https://www.gerritcodereview.com/)
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+unset GREP_OPTIONS
+
+CHANGE_ID_AFTER="Bug|Depends-On|Issue|Test|Feature|Fixes|Fixed"
+MSG="$1"
+
+# Check for, and add if missing, a unique Change-Id
+#
+add_ChangeId() {
+	clean_message=`sed -e '
+		/^diff --git .*/{
+			s///
+			q
+		}
+		/^Signed-off-by:/d
+		/^#/d
+	' "$MSG" | git stripspace`
+	if test -z "$clean_message"
+	then
+		return
+	fi
+
+	# Do not add Change-Id to temp commits
+	if echo "$clean_message" | head -1 | grep -q '^\(fixup\|squash\)!'
+	then
+		return
+	fi
+
+	if test "false" = "`git config --bool --get gerrit.createChangeId`"
+	then
+		return
+	fi
+
+	# Does Change-Id: already exist? if so, exit (no change).
+	if grep -i '^Change-Id:' "$MSG" >/dev/null
+	then
+		return
+	fi
+
+	id=`_gen_ChangeId`
+	T="$MSG.tmp.$$"
+	AWK=awk
+	if [ -x /usr/xpg4/bin/awk ]; then
+		# Solaris AWK is just too broken
+		AWK=/usr/xpg4/bin/awk
+	fi
+
+	# Get core.commentChar from git config or use default symbol
+	commentChar=`git config --get core.commentChar`
+	commentChar=${commentChar:-#}
+
+	# How this works:
+	# - parse the commit message as (textLine+ blankLine*)*
+	# - assume textLine+ to be a footer until proven otherwise
+	# - exception: the first block is not footer (as it is the title)
+	# - read textLine+ into a variable
+	# - then count blankLines
+	# - once the next textLine appears, print textLine+ blankLine* as these
+	#   aren't footer
+	# - in END, the last textLine+ block is available for footer parsing
+	$AWK '
+	BEGIN {
+		# while we start with the assumption that textLine+
+		# is a footer, the first block is not.
+		isFooter = 0
+		footerComment = 0
+		blankLines = 0
+	}
+
+	# Skip lines starting with commentChar without any spaces before it.
+	/^'"$commentChar"'/ { next }
+
+	# Skip the line starting with the diff command and everything after it,
+	# up to the end of the file, assuming it is only patch data.
+	# If more than one line before the diff was empty, strip all but one.
+	/^diff --git / {
+		blankLines = 0
+		while (getline) { }
+		next
+	}
+
+	# Count blank lines outside footer comments
+	/^$/ && (footerComment == 0) {
+		blankLines++
+		next
+	}
+
+	# Catch footer comment
+	/^\[[a-zA-Z0-9-]+:/ && (isFooter == 1) {
+		footerComment = 1
+	}
+
+	/]$/ && (footerComment == 1) {
+		footerComment = 2
+	}
+
+	# We have a non-blank line after blank lines. Handle this.
+	(blankLines > 0) {
+		print lines
+		for (i = 0; i < blankLines; i++) {
+			print ""
+		}
+
+		lines = ""
+		blankLines = 0
+		isFooter = 1
+		footerComment = 0
+	}
+
+	# Detect that the current block is not the footer
+	(footerComment == 0) && (!/^\[?[a-zA-Z0-9-]+:/ || /^[a-zA-Z0-9-]+:\/\//) {
+		isFooter = 0
+	}
+
+	{
+		# We need this information about the current last comment line
+		if (footerComment == 2) {
+			footerComment = 0
+		}
+		if (lines != "") {
+			lines = lines "\n";
+		}
+		lines = lines $0
+	}
+
+	# Footer handling:
+	# If the last block is considered a footer, splice in the Change-Id at the
+	# right place.
+	# Look for the right place to inject Change-Id by considering
+	# CHANGE_ID_AFTER. Keys listed in it (case insensitive) come first,
+	# then Change-Id, then everything else (eg. Signed-off-by:).
+	#
+	# Otherwise just print the last block, a new line and the Change-Id as a
+	# block of its own.
+	END {
+		unprinted = 1
+		if (isFooter == 0) {
+			print lines "\n"
+			lines = ""
+		}
+		changeIdAfter = "^(" tolower("'"$CHANGE_ID_AFTER"'") "):"
+		numlines = split(lines, footer, "\n")
+		for (line = 1; line <= numlines; line++) {
+			if (unprinted && match(tolower(footer[line]), changeIdAfter) != 1) {
+				unprinted = 0
+				print "Change-Id: I'"$id"'"
+			}
+			print footer[line]
+		}
+		if (unprinted) {
+			print "Change-Id: I'"$id"'"
+		}
+	}' "$MSG" > "$T" && mv "$T" "$MSG" || rm -f "$T"
+}
+_gen_ChangeIdInput() {
+	echo "tree `git write-tree`"
+	if parent=`git rev-parse "HEAD^0" 2>/dev/null`
+	then
+		echo "parent $parent"
+	fi
+	echo "author `git var GIT_AUTHOR_IDENT`"
+	echo "committer `git var GIT_COMMITTER_IDENT`"
+	echo
+	printf '%s' "$clean_message"
+}
+_gen_ChangeId() {
+	_gen_ChangeIdInput |
+	git hash-object -t commit --stdin
+}
+
+
+add_ChangeId

repo/hooks/pre-auto-gc

+#!/bin/sh
+#
+# An example hook script to verify if you are on battery, in case you
+# are running Windows, Linux or OS X. Called by git-gc --auto with no
+# arguments. The hook should exit with non-zero status after issuing an
+# appropriate message if it wants to stop the auto repacking.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+if uname -s | grep -q "_NT-"
+then
+	if test -x $SYSTEMROOT/System32/Wbem/wmic
+	then
+		STATUS=$(wmic path win32_battery get batterystatus /format:list | tr -d '\r\n')
+		[ "$STATUS" = "BatteryStatus=2" ] && exit 0 || exit 1
+	fi
+	exit 0
+fi
+
+if test -x /sbin/on_ac_power && (/sbin/on_ac_power;test $? -ne 1)
+then
+	exit 0
+elif test "$(cat /sys/class/power_supply/AC/online 2>/dev/null)" = 1
+then
+	exit 0
+elif grep -q 'on-line' /proc/acpi/ac_adapter/AC/state 2>/dev/null
+then
+	exit 0
+elif grep -q '0x01$' /proc/apm 2>/dev/null
+then
+	exit 0
+elif grep -q "AC Power \+: 1" /proc/pmu/info 2>/dev/null
+then
+	exit 0
+elif test -x /usr/bin/pmset && /usr/bin/pmset -g batt |
+	grep -q "drawing from 'AC Power'"
+then
+	exit 0
+elif test -d /sys/bus/acpi/drivers/battery && test 0 = \
+  "$(find /sys/bus/acpi/drivers/battery/ -type l | wc -l)";
+then
+	# No battery exists.
+	exit 0
+fi
+
+echo "Auto packing deferred; not on AC"
+exit 1

repo/pager.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2008 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import select
+import subprocess
+import sys
+
+import platform_utils
+
+active = False
+pager_process = None
+old_stdout = None
+old_stderr = None
+
+def RunPager(globalConfig):
+  if not os.isatty(0) or not os.isatty(1):
+    return
+  pager = _SelectPager(globalConfig)
+  if pager == '' or pager == 'cat':
+    return
+
+  if platform_utils.isWindows():
+    _PipePager(pager);
+  else:
+    _ForkPager(pager)
+
+def TerminatePager():
+  global pager_process, old_stdout, old_stderr
+  if pager_process:
+    sys.stdout.flush()
+    sys.stderr.flush()
+    pager_process.stdin.close()
+    pager_process.wait();
+    pager_process = None
+    # Restore initial stdout/err in case there is more output in this process
+    # after shutting down the pager process
+    sys.stdout = old_stdout
+    sys.stderr = old_stderr
+
+def _PipePager(pager):
+  global pager_process, old_stdout, old_stderr
+  assert pager_process is None, "Only one active pager process at a time"
+  # Create pager process, piping stdout/err into its stdin
+  pager_process = subprocess.Popen([pager], stdin=subprocess.PIPE, stdout=sys.stdout, stderr=sys.stderr)
+  old_stdout = sys.stdout
+  old_stderr = sys.stderr
+  sys.stdout = pager_process.stdin
+  sys.stderr = pager_process.stdin
+
+def _ForkPager(pager):
+  global active
+  # This process turns into the pager; a child it forks will
+  # do the real processing and output back to the pager. This
+  # is necessary to keep the pager in control of the tty.
+  #
+  try:
+    r, w = os.pipe()
+    pid = os.fork()
+    if not pid:
+      os.dup2(w, 1)
+      os.dup2(w, 2)
+      os.close(r)
+      os.close(w)
+      active = True
+      return
+
+    os.dup2(r, 0)
+    os.close(r)
+    os.close(w)
+
+    _BecomePager(pager)
+  except Exception:
+    print("fatal: cannot start pager '%s'" % pager, file=sys.stderr)
+    sys.exit(255)
+
+def _SelectPager(globalConfig):
+  try:
+    return os.environ['GIT_PAGER']
+  except KeyError:
+    pass
+
+  pager = globalConfig.GetString('core.pager')
+  if pager:
+    return pager
+
+  try:
+    return os.environ['PAGER']
+  except KeyError:
+    pass
+
+  return 'less'
+
+def _BecomePager(pager):
+  # Delaying execution of the pager until we have output
+  # ready works around a long-standing bug in popularly
+  # available versions of 'less', a better 'more'.
+  #
+  _a, _b, _c = select.select([0], [], [0])
+
+  os.environ['LESS'] = 'FRSX'
+
+  try:
+    os.execvp(pager, [pager])
+  except OSError:
+    os.execv('/bin/sh', ['sh', '-c', pager])

repo/platform_utils_win32.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+
+from ctypes import WinDLL, get_last_error, FormatError, WinError, addressof
+from ctypes import c_buffer
+from ctypes.wintypes import BOOL, BOOLEAN, LPCWSTR, DWORD, HANDLE, POINTER, c_ubyte
+from ctypes.wintypes import WCHAR, USHORT, LPVOID, Structure, Union, ULONG
+from ctypes.wintypes import byref
+
+kernel32 = WinDLL('kernel32', use_last_error=True)
+
+LPDWORD = POINTER(DWORD)
+UCHAR = c_ubyte
+
+# Win32 error codes
+ERROR_SUCCESS = 0
+ERROR_NOT_SUPPORTED = 50
+ERROR_PRIVILEGE_NOT_HELD = 1314
+
+# Win32 API entry points
+CreateSymbolicLinkW = kernel32.CreateSymbolicLinkW
+CreateSymbolicLinkW.restype = BOOLEAN
+CreateSymbolicLinkW.argtypes = (LPCWSTR,  # lpSymlinkFileName In
+                                LPCWSTR,  # lpTargetFileName In
+                                DWORD)    # dwFlags In
+
+# Symbolic link creation flags
+SYMBOLIC_LINK_FLAG_FILE = 0x00
+SYMBOLIC_LINK_FLAG_DIRECTORY = 0x01
+# symlink support for CreateSymbolicLink() starting with Windows 10 (1703, v10.0.14972)
+SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE = 0x02
+
+GetFileAttributesW = kernel32.GetFileAttributesW
+GetFileAttributesW.restype = DWORD
+GetFileAttributesW.argtypes = (LPCWSTR,)  # lpFileName In
+
+INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
+FILE_ATTRIBUTE_REPARSE_POINT = 0x00400
+
+CreateFileW = kernel32.CreateFileW
+CreateFileW.restype = HANDLE
+CreateFileW.argtypes = (LPCWSTR,  # lpFileName In
+                        DWORD,    # dwDesiredAccess In
+                        DWORD,    # dwShareMode In
+                        LPVOID,   # lpSecurityAttributes In_opt
+                        DWORD,    # dwCreationDisposition In
+                        DWORD,    # dwFlagsAndAttributes In
+                        HANDLE)   # hTemplateFile In_opt
+
+CloseHandle = kernel32.CloseHandle
+CloseHandle.restype = BOOL
+CloseHandle.argtypes = (HANDLE,)  # hObject In
+
+INVALID_HANDLE_VALUE = HANDLE(-1).value
+OPEN_EXISTING = 3
+FILE_FLAG_BACKUP_SEMANTICS = 0x02000000
+FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+
+DeviceIoControl = kernel32.DeviceIoControl
+DeviceIoControl.restype = BOOL
+DeviceIoControl.argtypes = (HANDLE,   # hDevice In
+                            DWORD,    # dwIoControlCode In
+                            LPVOID,   # lpInBuffer In_opt
+                            DWORD,    # nInBufferSize In
+                            LPVOID,   # lpOutBuffer Out_opt
+                            DWORD,    # nOutBufferSize In
+                            LPDWORD,  # lpBytesReturned Out_opt
+                            LPVOID)   # lpOverlapped Inout_opt
+
+# Device I/O control flags and options
+FSCTL_GET_REPARSE_POINT = 0x000900A8
+IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003
+IO_REPARSE_TAG_SYMLINK = 0xA000000C
+MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 0x4000
+
+
+class GENERIC_REPARSE_BUFFER(Structure):
+  _fields_ = (('DataBuffer', UCHAR * 1),)
+
+
+class SYMBOLIC_LINK_REPARSE_BUFFER(Structure):
+  _fields_ = (('SubstituteNameOffset', USHORT),
+              ('SubstituteNameLength', USHORT),
+              ('PrintNameOffset', USHORT),
+              ('PrintNameLength', USHORT),
+              ('Flags', ULONG),
+              ('PathBuffer', WCHAR * 1))
+
+  @property
+  def PrintName(self):
+    arrayt = WCHAR * (self.PrintNameLength // 2)
+    offset = type(self).PathBuffer.offset + self.PrintNameOffset
+    return arrayt.from_address(addressof(self) + offset).value
+
+
+class MOUNT_POINT_REPARSE_BUFFER(Structure):
+  _fields_ = (('SubstituteNameOffset', USHORT),
+              ('SubstituteNameLength', USHORT),
+              ('PrintNameOffset', USHORT),
+              ('PrintNameLength', USHORT),
+              ('PathBuffer', WCHAR * 1))
+
+  @property
+  def PrintName(self):
+    arrayt = WCHAR * (self.PrintNameLength // 2)
+    offset = type(self).PathBuffer.offset + self.PrintNameOffset
+    return arrayt.from_address(addressof(self) + offset).value
+
+
+class REPARSE_DATA_BUFFER(Structure):
+  class REPARSE_BUFFER(Union):
+    _fields_ = (('SymbolicLinkReparseBuffer', SYMBOLIC_LINK_REPARSE_BUFFER),
+                ('MountPointReparseBuffer', MOUNT_POINT_REPARSE_BUFFER),
+                ('GenericReparseBuffer', GENERIC_REPARSE_BUFFER))
+  _fields_ = (('ReparseTag', ULONG),
+              ('ReparseDataLength', USHORT),
+              ('Reserved', USHORT),
+              ('ReparseBuffer', REPARSE_BUFFER))
+  _anonymous_ = ('ReparseBuffer',)
+
+
+def create_filesymlink(source, link_name):
+  """Creates a Windows file symbolic link source pointing to link_name."""
+  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_FILE)
+
+
+def create_dirsymlink(source, link_name):
+  """Creates a Windows directory symbolic link source pointing to link_name.
+  """
+  _create_symlink(source, link_name, SYMBOLIC_LINK_FLAG_DIRECTORY)
+
+
+def _create_symlink(source, link_name, dwFlags):
+  if not CreateSymbolicLinkW(link_name, source, dwFlags | SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE):
+    # See https://github.com/golang/go/pull/24307/files#diff-b87bc12e4da2497308f9ef746086e4f0
+    # "the unprivileged create flag is unsupported below Windows 10 (1703, v10.0.14972).
+    # retry without it."
+    if not CreateSymbolicLinkW(link_name, source, dwFlags):
+      code = get_last_error()
+      error_desc = FormatError(code).strip()
+      if code == ERROR_PRIVILEGE_NOT_HELD:
+        raise OSError(errno.EPERM, error_desc, link_name)
+      _raise_winerror(
+          code,
+          'Error creating symbolic link \"%s\"'.format(link_name))
+
+
+def islink(path):
+  result = GetFileAttributesW(path)
+  if result == INVALID_FILE_ATTRIBUTES:
+    return False
+  return bool(result & FILE_ATTRIBUTE_REPARSE_POINT)
+
+
+def readlink(path):
+  reparse_point_handle = CreateFileW(path,
+                                     0,
+                                     0,
+                                     None,
+                                     OPEN_EXISTING,
+                                     FILE_FLAG_OPEN_REPARSE_POINT |
+                                     FILE_FLAG_BACKUP_SEMANTICS,
+                                     None)
+  if reparse_point_handle == INVALID_HANDLE_VALUE:
+    _raise_winerror(
+        get_last_error(),
+        'Error opening symblic link \"%s\"'.format(path))
+  target_buffer = c_buffer(MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
+  n_bytes_returned = DWORD()
+  io_result = DeviceIoControl(reparse_point_handle,
+                              FSCTL_GET_REPARSE_POINT,
+                              None,
+                              0,
+                              target_buffer,
+                              len(target_buffer),
+                              byref(n_bytes_returned),
+                              None)
+  CloseHandle(reparse_point_handle)
+  if not io_result:
+    _raise_winerror(
+        get_last_error(),
+        'Error reading symblic link \"%s\"'.format(path))
+  rdb = REPARSE_DATA_BUFFER.from_buffer(target_buffer)
+  if rdb.ReparseTag == IO_REPARSE_TAG_SYMLINK:
+    return _preserve_encoding(path, rdb.SymbolicLinkReparseBuffer.PrintName)
+  elif rdb.ReparseTag == IO_REPARSE_TAG_MOUNT_POINT:
+    return _preserve_encoding(path, rdb.MountPointReparseBuffer.PrintName)
+  # Unsupported reparse point type
+  _raise_winerror(
+      ERROR_NOT_SUPPORTED,
+      'Error reading symblic link \"%s\"'.format(path))
+
+
+def _preserve_encoding(source, target):
+  """Ensures target is the same string type (i.e. unicode or str) as source."""
+  if isinstance(source, unicode):
+    return unicode(target)
+  return str(target)
+
+
+def _raise_winerror(code, error_desc):
+  win_error_desc = FormatError(code).strip()
+  error_desc = "%s: %s".format(error_desc, win_error_desc)
+  raise WinError(code, error_desc)

repo/progress.py

+# -*- coding:utf-8 -*-
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from time import time
+from repo_trace import IsTrace
+
+_NOT_TTY = not os.isatty(2)
+
+# This will erase all content in the current line (wherever the cursor is).
+# It does not move the cursor, so this is usually followed by \r to move to
+# column 0.
+CSI_ERASE_LINE = '\x1b[2K'
+
+class Progress(object):
+  def __init__(self, title, total=0, units='', print_newline=False,
+               always_print_percentage=False):
+    self._title = title
+    self._total = total
+    self._done = 0
+    self._lastp = -1
+    self._start = time()
+    self._show = False
+    self._units = units
+    self._print_newline = print_newline
+    self._always_print_percentage = always_print_percentage
+
+  def update(self, inc=1):
+    self._done += inc
+
+    if _NOT_TTY or IsTrace():
+      return
+
+    if not self._show:
+      if 0.5 <= time() - self._start:
+        self._show = True
+      else:
+        return
+
+    if self._total <= 0:
+      sys.stderr.write('%s\r%s: %d,' % (
+        CSI_ERASE_LINE,
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+
+      if self._lastp != p or self._always_print_percentage:
+        self._lastp = p
+        sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s)%s' % (
+          CSI_ERASE_LINE,
+          self._title,
+          p,
+          self._done, self._units,
+          self._total, self._units,
+          "\n" if self._print_newline else ""))
+        sys.stderr.flush()
+
+  def end(self):
+    if _NOT_TTY or IsTrace() or not self._show:
+      return
+
+    if self._total <= 0:
+      sys.stderr.write('%s\r%s: %d, done.\n' % (
+        CSI_ERASE_LINE,
+        self._title,
+        self._done))
+      sys.stderr.flush()
+    else:
+      p = (100 * self._done) / self._total
+      sys.stderr.write('%s\r%s: %3d%% (%d%s/%d%s), done.\n' % (
+        CSI_ERASE_LINE,
+        self._title,
+        p,
+        self._done, self._units,
+        self._total, self._units))
+      sys.stderr.flush()