diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..0a86c1a75 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,98 @@ +# UNUSED, only for reference. If windows testing is needed, please add that to github actions +# CI on Windows via appveyor +environment: + GIT_DAEMON_PATH: "C:\\Program Files\\Git\\mingw64\\libexec\\git-core" + CYGWIN_GIT_PATH: "C:\\cygwin\\bin;%GIT_DAEMON_PATH%" + CYGWIN64_GIT_PATH: "C:\\cygwin64\\bin;%GIT_DAEMON_PATH%" + + matrix: + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + GIT_PATH: "%GIT_DAEMON_PATH%" + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + GIT_PATH: "%GIT_DAEMON_PATH%" + - PYTHON: "C:\\Python36-x64" + PYTHON_VERSION: "3.6" + GIT_PATH: "%GIT_DAEMON_PATH%" + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7" + GIT_PATH: "%GIT_DAEMON_PATH%" + - PYTHON: "C:\\Miniconda35-x64" + PYTHON_VERSION: "3.5" + IS_CONDA: "yes" + MAYFAIL: "yes" + GIT_PATH: "%GIT_DAEMON_PATH%" + ## Cygwin + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5" + IS_CYGWIN: "yes" + MAYFAIL: "yes" + GIT_PATH: "%CYGWIN64_GIT_PATH%" + +matrix: + allow_failures: + - MAYFAIL: "yes" +install: + - set PATH=%PYTHON%;%PYTHON%\Scripts;%GIT_PATH%;%PATH% + + ## Print configuration for debugging. + # + - | + echo %PATH% + uname -a + git --version + where git git-daemon python pip pip3 pip34 sh + python --version + python -c "import struct; print(struct.calcsize('P') * 8)" + + - IF "%IS_CONDA%" == "yes" ( + conda info -a & + conda install --yes --quiet pip + ) + - pip install -r requirements.txt + - pip install -r test-requirements.txt + - pip install codecov + + ## Copied from `init-tests-after-clone.sh`. + # + - | + git submodule update --init --recursive + git fetch --tags + git tag __testing_point__ + git checkout master || git checkout -b master + git reset --hard HEAD~1 + git reset --hard HEAD~1 + git reset --hard HEAD~1 + git reset --hard __testing_point__ + + ## For commits performed with the default user. + - | + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + + - pip install -e . + +build: false + +test_script: + - IF "%IS_CYGWIN%" == "yes" ( + nosetests -v + ) ELSE ( + IF "%PYTHON_VERSION%" == "3.5" ( + nosetests -v --with-coverage + ) ELSE ( + nosetests -v + ) + ) + +on_success: + - IF "%PYTHON_VERSION%" == "3.5" IF NOT "%IS_CYGWIN%" == "yes" (codecov) + +# Enable this to be able to login to the build worker. You can use the +# `remmina` program in Ubuntu, use the login information that the line below +# prints into the log. +#on_finish: +# - | +# echo "Running on_finish to establish connection back to the instance" +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) diff --git a/.codeclimate.yml b/.codeclimate.yml new file mode 100644 index 000000000..e658e6785 --- /dev/null +++ b/.codeclimate.yml @@ -0,0 +1,15 @@ +--- +engines: + duplication: + enabled: true + config: + languages: + - python + pep8: + enabled: true + radon: + enabled: true +ratings: + paths: + - "**.py" +exclude_paths: diff --git a/.deepsource.toml b/.deepsource.toml new file mode 100644 index 000000000..d55288b87 --- /dev/null +++ b/.deepsource.toml @@ -0,0 +1,15 @@ +version = 1 + +test_patterns = [ + 'test/**/test_*.py' +] + +exclude_patterns = [ + 'doc/**', + 'etc/sublime-text' +] + +[[analyzers]] +name = 'python' +enabled = true +runtime_version = '3.x.x' diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..b59962d21 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.git/ +.tox/ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..6d2618f2f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +test/fixtures/* eol=lf +init-tests-after-clone.sh diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 000000000..eb5c894e9 --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,59 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python package + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 9999 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies and prepare tests + run: | + set -x + python -m pip install --upgrade pip + python --version; git --version + git submodule update --init --recursive + git fetch --tags + + pip install -r test-requirements.txt + TRAVIS=yes ./init-tests-after-clone.sh + + git config --global user.email "travis@ci.com" + git config --global user.name "Travis Runner" + # If we rewrite the user's config by accident, we will mess it up + # and cause subsequent tests to fail + cat test/fixtures/.gitconfig >> ~/.gitconfig + - name: Lint with flake8 + run: | + set -x + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 --ignore=W293,E265,E266,W503,W504,E731 --count --show-source --statistics + - name: Test with nose + run: | + set -x + pip install nose + nosetests -v --with-coverage + - name: Documentation + run: | + set -x + pip install -r doc/requirements.txt + make -C doc html diff --git a/.gitignore b/.gitignore index ce8d0b1b9..db7c881cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.py[co] *.swp *~ +.venv/ +venv/ /*.egg-info /lib/GitPython.egg-info cover/ @@ -13,3 +15,9 @@ nbproject .DS_Store /*egg-info /.tox +/.vscode/ +.idea/ +.cache/ +.mypy_cache/ +.pytest_cache/ +monkeytype.sqlite3 diff --git a/.gitmodules b/.gitmodules index 4a3f37c25..251eeeec4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "gitdb"] - url = https://github.com/gitpython-developers/gitdb.git - path = git/ext/gitdb +[submodule "gitdb"] + url = https://github.com/gitpython-developers/gitdb.git + path = git/ext/gitdb diff --git a/.travis.private-ssh-key-for-issue-301 b/.travis.private-ssh-key-for-issue-301 deleted file mode 100644 index dd9c057b9..000000000 --- a/.travis.private-ssh-key-for-issue-301 +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEAu5awaLHx+F7O9gE1ac3gKztAi3OYzCS75XsUmQkjIZ+6je/3 -L+uVmtFyQ8t0PXaSF8rPBs+E35Vl1t9uz+vry7Q6bI5ClvlzaXCYHAuOqnDwrpdo -jykjKN4U0r0leDBO3rXpvjluKCGiE4pj0KgTe+xtoU4r64z7/DPQcaAmHbOFabxs -b8YZ/gHvA3vA8RA81eHWw83cw1U12dqjcE/gj6UZ0f4qj+PKQG6nfZEg6MzV71rS -wdHtCMljd4pz/3vzrCYCdl34jrzlm0lxuDbX/oMVe2OhoJAgilUQ6ieMhG0MKTr4 -Cf5LBRQGNKkuwNgj70fmsYphm3xKeehjvwxB/wIDAQABAoIBACKZkMA+0hq1ogbO -AkQZrRdMPoN6Ntywidex9AKmJMy24Xn8oOM92MJNj33hsPcFP70Ts0vWSvPSYo5X -d+Dx6vQPshcDxlSCfj8cfXHuz8mwOzR4yNhfBhUroTudl6OrhWCevRZREXhle7eO -9wotdiqNWUs/V+qCfpfuFqxelak0erSSdR4rh35uZYNBdopPiMlo8klYEDRGrH2R -LCjeOiC9pkkj7f1ylyDCjkI8hdjOnWO/QYYD7g8ZDKFwRRBm0os6d3Nj20ED7N8y -eBRBjDv6k5165MAtFmjHrO9olUh1o+2rwrYK+hVckg5uE/cv6VanRs9tRgF6TyIw -97tSeQECgYEA3yuy0aINAW+T7+3oOPuo9c6ZrBMrPTOlRJufrLS4WvV3laL/AVNt -DmKkQOjKbQI8cknlqV9CovteRT2Dj9lTZwSGmecVPijslAUmwz9s4Ucu1BR8Qckl -i470z5Fge6fdPsAYhjhqjbLUE2Lml2A/YyNnpTFSP4wXpyy6HLkT1pECgYEA1y8E -0zVlnoeHKDgpGDpmXV9YhcqtQ6Q2HC/g9oUn+jAfeYkVX8L5brUX2S4L03hYAljH -j0RJZnOZoXpN6WMRnWJkOP80Fkq5Jv6vJkghNPJ7vWtXJan4wkNr/nGQ65DvxeLJ -qIEmSe+sZHCB/W26y9bgI+ELHt4VgPuEIoGad48CgYBNza5J53UhSwUIYKjVNOdy -ytRIaWZpr9euU5MXYunizDEkue6tR6h9m2YoOwBXgLASKDEpG0zgBUKYYRm8zMeG -4s0KWsXNJfdUo3cgGryazXZF+d5YEQhF31D6DHTWp286sT3bjU+Ylv/YwmIh5Cw1 -I+K+dLN39B9K6Qz5doy0AQKBgFHlzFFSflWiwVcWYNWezHz3H+rz45Pd+NYrhtRs -g3WeQSxxdxgWTfbLp8L52nhm0iA6h+FIHSOIFc22jdao5PhgjUKAJuPhFqz3u9O3 -GiRbsaq7ItTr5wiQvpZ9xKlTZV6MXTHrzZXQSW2EwISi8bhJFM+zBBeAcPDBSV4U -s6STAoGANTuc66z/2wtZYINA2+JN1kSELL22ka8yKYWBmi0NJ7Hp+7MPKj9v71mM -Oq4zuNIYEJfQaf53xhi6BhZVWMERDKMGWqJet88BATbBIqBahHxFTiCmsP6fBH7e -AipTene7xg9mqG8qoluFdHRvzGqZFu0lZSGxtZYeU/y+U2b7lzI= ------END RSA PRIVATE KEY----- diff --git a/.travis.yml b/.travis.yml index c68a24b83..1fbb1ddb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,30 @@ +# UNUSED, only for reference. If adjustments are needed, please see github actions language: python python: - - "2.6" - - "2.7" - - "3.3" - "3.4" + - "3.5" + - "3.6" + - "3.7" + - "3.8" + - "nightly" # - "pypy" - won't work as smmap doesn't work (see gitdb/.travis.yml for details) +matrix: + allow_failures: + - python: "nightly" git: # a higher depth is needed for most of the tests - must be high enough to not actually be shallow # as we clone our own repository in the process depth: 99999 install: + - python --version; git --version - git submodule update --init --recursive - git fetch --tags - - pip install coveralls flake8 sphinx + - pip install -r test-requirements.txt + - pip install -r doc/requirements.txt + - pip install codecov # generate some reflog as git-python tests need it (in master) - - git tag __testing_point__ - - git checkout master - - git reset --hard HEAD~1 - - git reset --hard HEAD~1 - - git reset --hard HEAD~1 - - git reset --hard __testing_point__ + - ./init-tests-after-clone.sh # as commits are performed with the default user, it needs to be set for travis too - git config --global user.email "travis@ci.com" @@ -28,19 +32,13 @@ install: # If we rewrite the user's config by accident, we will mess it up # and cause subsequent tests to fail - cat git/test/fixtures/.gitconfig >> ~/.gitconfig - # used later when testing for specific issue - # copy the configured key into place so clone can work - # The key just has read-only access to the test-repository - - cp .travis.private-ssh-key-for-issue-301 ~/.ssh/id_rsa - - chmod 0400 ~/.ssh/id_rsa - - git clone git@github.com:gitpython-developers/gitpython_issue_301.git issue-301-repo -script: +script: # Make sure we limit open handles to see if we are leaking them - - ulimit -n 96 + - ulimit -n 128 - ulimit -n - - nosetests -v --with-coverage - - '( cd issue-301-repo && PYTHONPATH=$PWD/.. ./reproduce.py )' - - flake8 - - cd doc && make html + - coverage run --omit="test/*" -m unittest --buffer + - coverage report + - if [ "$TRAVIS_PYTHON_VERSION" == '3.5' ]; then cd doc && make html; fi + - if [ "$TRAVIS_PYTHON_VERSION" == '3.6' ]; then flake8 --ignore=W293,E265,E266,W503,W504,E731; fi after_success: - - coveralls + - codecov diff --git a/AUTHORS b/AUTHORS index 40fa69883..7b21b2b26 100644 --- a/AUTHORS +++ b/AUTHORS @@ -12,5 +12,35 @@ Contributors are: -Kai Lautaportti -Paul Sowden -Sebastian Thiel - +-Jonathan Chu +-Vincent Driessen +-Phil Elson +-Bernard `Guyzmo` Pratz +-Timothy B. Hartman +-Konstantin Popov +-Peter Jones +-Anson Mansfield +-Ken Odegard +-Alexis Horgix Chotard +-Piotr Babij +-Mikuláš Poul +-Charles Bouchard-Légaré +-Yaroslav Halchenko +-Tim Swast +-William Luc Ritchie +-David Host +-A. Jesse Jiryu Davis +-Steven Whitman +-Stefan Stancu +-César Izurieta +-Arthur Milchior +-Anil Khatri +-JJ Graham +-Ben Thayer +-Dries Kennes +-Pratik Anurag +-Harmon +-Liam Beguin +-Ram Rachum +-Alba Mendez Portions derived from other open source works and are clearly marked. diff --git a/CHANGES b/CHANGES index 9242253ff..aa8116b23 100644 --- a/CHANGES +++ b/CHANGES @@ -1,2 +1,2 @@ Please see the online documentation for the latest changelog: -https://github.com/gitpython-developers/GitPython/blob/0.3/doc/source/changes.rst +https://github.com/gitpython-developers/GitPython/blob/master/doc/source/changes.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..4217cbaf9 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,10 @@ +### How to contribute + +The following is a short step-by-step rundown of what one typically would do to contribute. + +* [fork this project](https://github.com/gitpython-developers/GitPython/fork) on GitHub. +* For setting up the environment to run the self tests, please look at `.travis.yml`. +* Please try to **write a test that fails unless the contribution is present.** +* Feel free to add yourself to AUTHORS file. +* Create a pull request. + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..f2d7e22f5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,84 @@ +# +# Contributed by: James E. King III (@jeking3) +# +# This Dockerfile creates an Ubuntu Xenial build environment +# that can run the same test suite as Travis CI. +# + +FROM ubuntu:xenial + +# Metadata +LABEL maintainer="jking@apache.org" +LABEL description="CI environment for testing GitPython" + +ENV CONTAINER_USER=user +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + add-apt-key \ + apt \ + apt-transport-https \ + apt-utils \ + ca-certificates \ + curl \ + git \ + net-tools \ + openssh-client \ + sudo \ + vim \ + wget + +RUN add-apt-key -v 6A755776 -k keyserver.ubuntu.com && \ + add-apt-key -v E1DF1F24 -k keyserver.ubuntu.com && \ + echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu xenial main" >> /etc/apt/sources.list && \ + echo "deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu xenial main" >> /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y --install-recommends git python2.7 python3.4 python3.5 python3.6 python3.7 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python2.7 27 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.4 34 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.5 35 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 36 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 37 + +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py && \ + pip3 install tox + +# Clean up +RUN rm -rf /var/cache/apt/* && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/* && \ + rm -rf /var/tmp/* + +################################################################# +# Build as a regular user +# Credit: https://github.com/delcypher/docker-ubuntu-cxx-dev/blob/master/Dockerfile +# License: None specified at time of import +# Add non-root user for container but give it sudo access. +# Password is the same as the username +RUN useradd -m ${CONTAINER_USER} && \ + echo ${CONTAINER_USER}:${CONTAINER_USER} | chpasswd && \ + echo "${CONTAINER_USER} ALL=(root) ALL" >> /etc/sudoers +RUN chsh --shell /bin/bash ${CONTAINER_USER} +USER ${CONTAINER_USER} +################################################################# + +# The test suite will not tolerate running against a branch that isn't "master", so +# check out the project to a well-known location that can be used by the test suite. +# This has the added benefit of protecting the local repo fed into the container +# as a volume from getting destroyed by a bug exposed by the test suite. :) +ENV TRAVIS=ON +RUN git clone --recursive https://github.com/gitpython-developers/GitPython.git /home/${CONTAINER_USER}/testrepo && \ + cd /home/${CONTAINER_USER}/testrepo && \ + ./init-tests-after-clone.sh +ENV GIT_PYTHON_TEST_GIT_REPO_BASE=/home/${CONTAINER_USER}/testrepo +ENV TRAVIS= + +# Ensure any local pip installations get on the path +ENV PATH=/home/${CONTAINER_USER}/.local/bin:${PATH} + +# Set the global default git user to be someone non-descript +RUN git config --global user.email ci@gitpython.org && \ + git config --global user.name "GitPython CI User" + diff --git a/MANIFEST.in b/MANIFEST.in index c84a9dd32..5fd771db3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,8 +2,11 @@ include VERSION include LICENSE include CHANGES include AUTHORS -include README +include CONTRIBUTING.md +include README.md include requirements.txt -graft git/test/fixtures -graft git/test/performance +recursive-include doc * +recursive-exclude test * + +global-exclude __pycache__ *.pyc diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..709813ff2 --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ +.PHONY: all clean release force_release docker-build test nose-pdb + +all: + @grep -Ee '^[a-z].*:' Makefile | cut -d: -f1 | grep -vF all + +clean: + rm -rf build/ dist/ .eggs/ .tox/ + +release: clean + # Check if latest tag is the current head we're releasing + echo "Latest tag = $$(git tag | sort -nr | head -n1)" + echo "HEAD SHA = $$(git rev-parse head)" + echo "Latest tag SHA = $$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + @test "$$(git rev-parse head)" = "$$(git tag | sort -nr | head -n1 | xargs git rev-parse)" + make force_release + +force_release: clean + git push --tags origin master + python3 setup.py sdist bdist_wheel + twine upload -s -i 27C50E7F590947D7273A741E85194C08421980C9 dist/* + +docker-build: + docker build --quiet -t gitpython:xenial -f Dockerfile . + +test: docker-build + # NOTE!!! + # NOTE!!! If you are not running from master or have local changes then tests will fail + # NOTE!!! + docker run --rm -v ${CURDIR}:/src -w /src -t gitpython:xenial tox + +nose-pdb: docker-build + # run tests under nose and break on error or failure into python debugger + # HINT: set PYVER to "pyXX" to change from the default of py37 to pyXX for nose tests + docker run --rm --env PYVER=${PYVER} -v ${CURDIR}:/src -w /src -it gitpython:xenial /bin/bash dockernose.sh diff --git a/README.md b/README.md index 75fcdcb2c..0d0edeb43 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,53 @@ +## [Gitoxide](https://github.com/Byron/gitoxide): A peek into the future… + +I started working on GitPython in 2009, back in the days when Python was 'my thing' and I had great plans with it. +Of course, back in the days, I didn't really know what I was doing and this shows in many places. Somewhat similar to +Python this happens to be 'good enough', but at the same time is deeply flawed and broken beyond repair. + +By now, GitPython is widely used and I am sure there is a good reason for that, it's something to be proud of and happy about. +The community is maintaining the software and is keeping it relevant for which I am absolutely grateful. For the time to come I am happy to continue maintaining GitPython, remaining hopeful that one day it won't be needed anymore. + +More than 15 years after my first meeting with 'git' I am still in excited about it, and am happy to finally have the tools and +probably the skills to scratch that itch of mine: implement `git` in a way that makes tool creation a piece of cake for most. + +If you like the idea and want to learn more, please head over to [gitoxide](https://github.com/Byron/gitoxide), an +implementation of 'git' in [Rust](https://www.rust-lang.org). + ## GitPython -GitPython is a python library used to interact with git repositories, high-level like git-porcelain, or low-level like git-plumbing. +GitPython is a python library used to interact with git repositories, high-level like git-porcelain, +or low-level like git-plumbing. -It provides abstractions of git objects for easy access of repository data, and additionally allows you to access the git repository more directly using either a pure python implementation, or the faster, but more resource intensive git command implementation. +It provides abstractions of git objects for easy access of repository data, and additionally +allows you to access the git repository more directly using either a pure python implementation, +or the faster, but more resource intensive *git command* implementation. + +The object database implementation is optimized for handling large quantities of objects and large datasets, +which is achieved by using low-level structures and data streaming. -The object database implementation is optimized for handling large quantities of objects and large datasets, which is achieved by using low-level structures and data streaming. ### REQUIREMENTS -GitPython needs the `git` executable to be installed on the system and available in your `PATH` for most operations. If it is not in your `PATH`, you can help GitPython find it by setting the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. +GitPython needs the `git` executable to be installed on the system and available +in your `PATH` for most operations. +If it is not in your `PATH`, you can help GitPython find it by setting +the `GIT_PYTHON_GIT_EXECUTABLE=` environment variable. * Git (1.7.x or newer) +* Python >= 3.5 -The list of dependencies are listed in `./requirements.txt` and `./test-requirements.txt`. The installer takes care of installing them for you. +The list of dependencies are listed in `./requirements.txt` and `./test-requirements.txt`. +The installer takes care of installing them for you. ### INSTALL -[![Latest Version](https://pypip.in/version/GitPython/badge.svg)](https://pypi.python.org/pypi/GitPython/) -[![Supported Python Versions](https://pypip.in/py_versions/GitPython/badge.svg)](https://pypi.python.org/pypi/GitPython/) - If you have downloaded the source code: python setup.py install or if you want to obtain a copy from the Pypi repository: - pip install gitpython + pip install GitPython Both commands will install the required package dependencies. @@ -33,9 +55,45 @@ A distribution package can be obtained for manual installation at: http://pypi.python.org/pypi/GitPython +If you like to clone from source, you can do it like so: + +```bash +git clone https://github.com/gitpython-developers/GitPython +git submodule update --init --recursive +./init-tests-after-clone.sh +``` + +### Limitations + +#### Leakage of System Resources + +GitPython is not suited for long-running processes (like daemons) as it tends to +leak system resources. It was written in a time where destructors (as implemented +in the `__del__` method) still ran deterministically. + +In case you still want to use it in such a context, you will want to search the +codebase for `__del__` implementations and call these yourself when you see fit. + +Another way assure proper cleanup of resources is to factor out GitPython into a +separate process which can be dropped periodically. + +#### Windows support + +See [Issue #525](https://github.com/gitpython-developers/GitPython/issues/525). + ### RUNNING TESTS -The easiest way to run test is by using [tox](https://pypi.python.org/pypi/tox) a wrapper around virtualenv. It will take care of setting up environnements with the proper dependencies installed and execute test commands. To install it simply: +*Important*: Right after cloning this repository, please be sure to have executed +the `./init-tests-after-clone.sh` script in the repository root. Otherwise +you will encounter test failures. + +On *Windows*, make sure you have `git-daemon` in your PATH. For MINGW-git, the `git-daemon.exe` +exists in `Git\mingw64\libexec\git-core\`; CYGWIN has no daemon, but should get along fine +with MINGW's. + +The easiest way to run tests is by using [tox](https://pypi.python.org/pypi/tox) +a wrapper around virtualenv. It will take care of setting up environments with the proper +dependencies installed and execute test commands. To install it simply: pip install tox @@ -43,49 +101,116 @@ Then run: tox -### SOURCE - -GitPython's git repo is available on GitHub, which can be browsed at [github](https://github.com/gitpython-developers/GitPython) and cloned like that: - git clone https://github.com/gitpython-developers/GitPython +For more fine-grained control, you can use `unittest`. -### Live Coding +### Contributions -You can watch me fix issues or implement new features [live on Twitch][twitch-channel], or have a look at [past recordings on youtube][youtube-playlist] - -* [Live on Twitch][twitch-channel] (just follow the channel to be notified when a session starts) -* [Archive on Youtube][youtube-playlist] +Please have a look at the [contributions file][contributing]. ### INFRASTRUCTURE * [User Documentation](http://gitpython.readthedocs.org) * [Questions and Answers](http://stackexchange.com/filters/167317/gitpython) * Please post on stackoverflow and use the `gitpython` tag -* [Mailing List](http://groups.google.com/group/git-python) - * Please use it for everything that doesn't fit Stackoverflow. * [Issue Tracker](https://github.com/gitpython-developers/GitPython/issues) - * Post reproducible bugs and feature requests as a new issue. Please be sure to provide the following information if posting bugs: + * Post reproducible bugs and feature requests as a new issue. + Please be sure to provide the following information if posting bugs: * GitPython version (e.g. `import git; git.__version__`) * Python version (e.g. `python --version`) * The encountered stack-trace, if applicable * Enough information to allow reproducing the issue +### How to make a new release + +* Update/verify the **version** in the `VERSION` file +* Update/verify that the `doc/source/changes.rst` changelog file was updated +* Commit everything +* Run `git tag -s ` to tag the version in Git +* Run `make release` +* Close the milestone mentioned in the _changelog_ and create a new one. _Do not reuse milestones by renaming them_. +* set the upcoming version in the `VERSION` file, usually be + incrementing the patch level, and possibly by appending `-dev`. Probably you + want to `git push` once more. + +### How to verify a release + +Please only use releases from `pypi` as you can verify the respective source +tarballs. + +This script shows how to verify the tarball was indeed created by the authors of +this project: + +``` +curl https://files.pythonhosted.org/packages/09/bc/ae32e07e89cc25b9e5c793d19a1e5454d30a8e37d95040991160f942519e/GitPython-3.1.8-py3-none-any.whl > gitpython.whl +curl https://files.pythonhosted.org/packages/09/bc/ae32e07e89cc25b9e5c793d19a1e5454d30a8e37d95040991160f942519e/GitPython-3.1.8-py3-none-any.whl.asc > gitpython-signature.asc +gpg --verify gitpython-signature.asc gitpython.whl +``` + +which outputs + +``` +gpg: Signature made Fr 4 Sep 10:04:50 2020 CST +gpg: using RSA key 27C50E7F590947D7273A741E85194C08421980C9 +gpg: Good signature from "Sebastian Thiel (YubiKey USB-C) " [ultimate] +gpg: aka "Sebastian Thiel (In Rust I trust) " [ultimate] +``` + +You can verify that the keyid indeed matches the release-signature key provided in this +repository by looking at the keys details: + +``` +gpg --list-packets ./release-verification-key.asc +``` + +You can verify that the commit adding it was also signed by it using: + +``` +git show --show-signature ./release-verification-key.asc +``` + +If you would like to trust it permanently, you can import and sign it: + +``` +gpg --import ./release-verification-key.asc +gpg --edit-key 4C08421980C9 + +> sign +> save +``` + +### Projects using GitPython + +* [PyDriller](https://github.com/ishepard/pydriller) +* [Kivy Designer](https://github.com/kivy/kivy-designer) +* [Prowl](https://github.com/nettitude/Prowl) +* [Python Taint](https://github.com/python-security/pyt) +* [Buster](https://github.com/axitkhurana/buster) +* [git-ftp](https://github.com/ezyang/git-ftp) +* [Git-Pandas](https://github.com/wdm0006/git-pandas) +* [PyGitUp](https://github.com/msiemens/PyGitUp) +* [PyJFuzz](https://github.com/mseclab/PyJFuzz) +* [Loki](https://github.com/Neo23x0/Loki) +* [Omniwallet](https://github.com/OmniLayer/omniwallet) +* [GitViper](https://github.com/BeayemX/GitViper) +* [Git Gud](https://github.com/bthayer2365/git-gud) + ### LICENSE New BSD License. See the LICENSE file. ### DEVELOPMENT STATUS -[![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg)](https://travis-ci.org/gitpython-developers/GitPython) -[![Code Climate](https://codeclimate.com/github/gitpython-developers/GitPython/badges/gpa.svg)](https://codeclimate.com/github/gitpython-developers/GitPython) +![Python package](https://github.com/gitpython-developers/GitPython/workflows/Python%20package/badge.svg) [![Documentation Status](https://readthedocs.org/projects/gitpython/badge/?version=stable)](https://readthedocs.org/projects/gitpython/?badge=stable) -[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/GitPython/badge/pr)](http://www.issuestats.com/github/gitpython-developers/GitPython) -[![Issue Stats](http://www.issuestats.com/github/gitpython-developers/GitPython/badge/issue)](http://www.issuestats.com/github/gitpython-developers/GitPython) +[![Packaging status](https://repology.org/badge/tiny-repos/python:gitpython.svg)](https://repology.org/metapackage/python:gitpython/versions) + +This project is in **maintenance mode**, which means that -Now that there seems to be a massive user base, this should be motivation enough to let git-python return to a proper state, which means +* …there will be no feature development, unless these are contributed +* …there will be no bug fixes, unless they are relevant to the safety of users, or contributed +* …issues will be responded to with waiting times of up to a month -* no open pull requests -* no open issues describing bugs +The project is open to contributions of all kinds, as well as new maintainers. -[twitch-channel]: http://www.twitch.tv/byronimo/profile -[youtube-playlist]: https://www.youtube.com/playlist?list=PLMHbQxe1e9MnoEcLhn6Yhv5KAvpWkJbL0 \ No newline at end of file +[contributing]: https://github.com/gitpython-developers/GitPython/blob/master/CONTRIBUTING.md diff --git a/TODO b/TODO deleted file mode 100644 index 2643676ce..000000000 --- a/TODO +++ /dev/null @@ -1,7 +0,0 @@ -For a list of tickets, please visit -http://byronimo.lighthouseapp.com/projects/51787-gitpython/overview - - - - - diff --git a/VERSION b/VERSION index 7dea76edb..2a399f7d1 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.1 +3.1.14 diff --git a/doc/Makefile b/doc/Makefile index 39fe377f9..ef2d60e5f 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -W SPHINXBUILD = sphinx-build PAPER = diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 000000000..98e5c06a0 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,2 @@ +sphinx<2.0 +sphinx_rtd_theme diff --git a/doc/source/changes.rst b/doc/source/changes.rst index e6d7b09be..93f65a2f7 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,10 +2,449 @@ Changelog ========= +3.1.15 (UNRELEASED) +=================== + +* add deprectation warning for python 3.5 + +3.1.14 +====== + +* git.Commit objects now have a ``replace`` method that will return a + copy of the commit with modified attributes. +* Add python 3.9 support +* Drop python 3.4 support + +3.1.13 +====== + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/45?closed=1 + +3.1.12 +====== + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/44?closed=1 + +3.1.11 +====== + +Fixes regression of 3.1.10. + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/43?closed=1 + +3.1.10 +====== + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/42?closed=1 + + +3.1.9 +===== + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/41?closed=1 + + +3.1.8 +===== + +* support for 'includeIf' in git configuration files +* tests are now excluded from the package, making it conisderably smaller + + +See the following for more details: +https://github.com/gitpython-developers/gitpython/milestone/40?closed=1 + + +3.1.7 +===== + +* Fix tutorial examples, which disappeared in 3.1.6 due to a missed path change. + +3.1.6 +===== + +* Greatly reduced package size, see https://github.com/gitpython-developers/GitPython/pull/1031 + +3.1.5 +===== + +* rollback: package size was reduced significantly not placing tests into the package anymore. + See https://github.com/gitpython-developers/GitPython/issues/1030 + +3.1.4 +===== + +* all exceptions now keep track of their cause +* package size was reduced significantly not placing tests into the package anymore. + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/39?closed=1 + +3.1.3 +===== + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/38?closed=1 + +3.1.2 +===== + +* Re-release of 3.1.1, with known signature + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/37?closed=1 + + +3.1.1 +===== + +* support for PyOxidizer, which previously failed due to usage of `__file__`. + +See the following for details: +https://github.com/gitpython-developers/gitpython/milestone/36?closed=1 + + +3.1.0 +===== + +* Switched back to using gitdb package as requirement + (`gitdb#59 `_) + +3.0.9 +===== + +* Restricted GitDB (gitdb2) version requirement to < 4 +* Removed old nose library from test requirements + +Bugfixes +-------- + +* Changed to use UTF-8 instead of default encoding when getting information about a symbolic reference + (`#774 `_) +* Fixed decoding of tag object message so as to replace invalid bytes + (`#943 `_) + +3.0.8 +===== + +* Added support for Python 3.8 +* Bumped GitDB (gitdb2) version requirement to > 3 + +Bugfixes +-------- + +* Fixed Repo.__repr__ when subclassed + (`#968 `_) +* Removed compatibility shims for Python < 3.4 and old mock library +* Replaced usage of deprecated unittest aliases and Logger.warn +* Removed old, no longer used assert methods +* Replaced usage of nose assert methods with unittest + +3.0.7 +===== + +Properly signed re-release of v3.0.6 with new signature +(See `#980 `_) + +3.0.6 +===== + +| Note: There was an issue that caused this version to be released to PyPI without a signature +| See the changelog for v3.0.7 and `#980 `_ + +Bugfixes +-------- + +* Fixed warning for usage of environment variables for paths containing ``$`` or ``%`` + (`#832 `_, + `#961 `_) +* Added support for parsing Git internal date format (@ ) + (`#965 `_) +* Removed Python 2 and < 3.3 compatibility shims + (`#979 `_) +* Fixed GitDB (gitdb2) requirement version specifier formatting in requirements.txt + (`#979 `_) + +3.0.5 - Bugfixes +============================================= + +see the following for details: +https://github.com/gitpython-developers/gitpython/milestone/32?closed=1 + +3.0.4 - Bugfixes +============================================= + +see the following for details: +https://github.com/gitpython-developers/gitpython/milestone/31?closed=1 + +3.0.3 - Bugfixes +============================================= + +see the following for (most) details: +https://github.com/gitpython-developers/gitpython/milestone/30?closed=1 + +3.0.2 - Bugfixes +============================================= + +* fixes an issue with installation + +3.0.1 - Bugfixes and performance improvements +============================================= + +* Fix a `performance regression `__ which could make certain workloads 50% slower +* Add `currently_rebasing_on` method on `Repo`, see `the PR `__ +* Fix incorrect `requirements.txt` which could lead to broken installations, see this `issue `__ for details. + +3.0.0 - Remove Python 2 support +=============================== + +Motivation for this is a patch which improves unicode handling when dealing with filesystem paths. +Python 2 compatibility was introduced to deal with differences, and I thought it would be a good idea +to 'just' drop support right now, mere 5 months away from the official maintenance stop of python 2.7. + +The underlying motivation clearly is my anger when thinking python and unicode, which was a hassle from the +start, at least in a codebase as old as GitPython, which totally doesn't handle encodings correctly in many cases. + +Having migrated to using `Rust` exclusively for tooling, I still see that correct handling of encodings isn't entirely +trivial, but at least `Rust` makes clear what has to be done at compile time, allowing to write software that is pretty +much guaranteed to work once it compiles. + +Again, my apologies if removing Python 2 support caused inconveniences, please see release 2.1.13 which returns it. + +see the following for (most) details: +https://github.com/gitpython-developers/gitpython/milestone/27?closed=1 + +or run have a look at the difference between tags v2.1.12 and v3.0.0: +https://github.com/gitpython-developers/GitPython/compare/2.1.12...3.0.0. + +2.1.15 +====== + +* Fixed GitDB (gitdb2) requirement version specifier formatting in requirements.txt + (Backported from `#979 `_) +* Restricted GitDB (gitdb2) version requirement to < 3 + (`#897 `_) + +2.1.14 +====== + +* Fixed handling of 0 when transforming kwargs into Git command arguments + (Backported from `#899 `_) + +2.1.13 - Bring back Python 2.7 support +====================================== + +My apologies for any inconvenience this may have caused. Following semver, backward incompatible changes +will be introduced in a minor version. + +2.1.12 - Bugfixes and Features +============================== + +* Multi-value support and interface improvements for Git configuration. Thanks to A. Jesse Jiryu Davis. + +or run have a look at the difference between tags v2.1.11 and v2.1.12: +https://github.com/gitpython-developers/GitPython/compare/2.1.11...2.1.12 + +2.1.11 - Bugfixes +================= + +see the following for (most) details: +https://github.com/gitpython-developers/gitpython/milestone/26?closed=1 + +or run have a look at the difference between tags v2.1.10 and v2.1.11: +https://github.com/gitpython-developers/GitPython/compare/2.1.10...2.1.11 + +2.1.10 - Bugfixes +================= + +see the following for (most) details: +https://github.com/gitpython-developers/gitpython/milestone/25?closed=1 + +or run have a look at the difference between tags v2.1.9 and v2.1.10: +https://github.com/gitpython-developers/GitPython/compare/2.1.9...2.1.10 + +2.1.9 - Dropping support for Python 2.6 +======================================= + +see the following for (most) details: +https://github.com/gitpython-developers/gitpython/milestone/24?closed=1 + +or run have a look at the difference between tags v2.1.8 and v2.1.9: +https://github.com/gitpython-developers/GitPython/compare/2.1.8...2.1.9 + + +2.1.8 - bugfixes +==================================== + +see the following for (most) details: +https://github.com/gitpython-developers/gitpython/milestone/23?closed=1 + +or run have a look at the difference between tags v2.1.7 and v2.1.8: +https://github.com/gitpython-developers/GitPython/compare/2.1.7...2.1.8 + +2.1.6 - bugfixes +==================================== + +* support for worktrees + +2.1.3 - Bugfixes +==================================== + +All issues and PRs can be viewed in all detail when following this URL: +https://github.com/gitpython-developers/GitPython/milestone/21?closed=1 + + +2.1.1 - Bugfixes +==================================== + +All issues and PRs can be viewed in all detail when following this URL: +https://github.com/gitpython-developers/GitPython/issues?q=is%3Aclosed+milestone%3A%22v2.1.1+-+Bugfixes%22 + + +2.1.0 - Much better windows support! +==================================== + +Special thanks to @ankostis, who made this release possible (nearly) single-handedly. +GitPython is run by its users, and their PRs make all the difference, they keep +GitPython relevant. Thank you all so much for contributing ! + +Notable fixes +------------- + +* The `GIT_DIR` environment variable does not override the `path` argument when + initializing a `Repo` object anymore. However, if said `path` unset, `GIT_DIR` + will be used to fill the void. + +All issues and PRs can be viewed in all detail when following this URL: +https://github.com/gitpython-developers/GitPython/issues?q=is%3Aclosed+milestone%3A%22v2.1.0+-+proper+windows+support%22 + + +2.0.9 - Bugfixes +============================= + +* `tag.commit` will now resolve commits deeply. +* `Repo` objects can now be pickled, which helps with multi-processing. +* `Head.checkout()` now deals with detached heads, which is when it will return + the `HEAD` reference instead. + +* `DiffIndex.iter_change_type(...)` produces better results when diffing + +2.0.8 - Features and Bugfixes +============================= + +* `DiffIndex.iter_change_type(...)` produces better results when diffing + an index against the working tree. +* `Repo().is_dirty(...)` now supports the `path` parameter, to specify a single + path by which to filter the output. Similar to `git status ` +* Symbolic refs created by this library will now be written with a newline + character, which was previously missing. +* `blame()` now properly preserves multi-line commit messages. +* No longer corrupt ref-logs by writing multi-line comments into them. + +2.0.7 - New Features +==================== + +* `IndexFile.commit(...,skip_hooks=False)` added. This parameter emulates the + behaviour of `--no-verify` on the command-line. + +2.0.6 - Fixes and Features +========================== + +* Fix: remote output parser now correctly matches refs with non-ASCII + chars in them +* API: Diffs now have `a_rawpath`, `b_rawpath`, `raw_rename_from`, + `raw_rename_to` properties, which are the raw-bytes equivalents of their + unicode path counterparts. +* Fix: TypeError about passing keyword argument to string decode() on + Python 2.6. +* Feature: `setUrl API on Remotes `__ + +2.0.5 - Fixes +============= + +* Fix: parser of fetch info lines choked on some legitimate lines + +2.0.4 - Fixes +============= + +* Fix: parser of commit object data is now robust against cases where + commit object contains invalid bytes. The invalid characters are now + replaced rather than choked on. +* Fix: non-ASCII paths are now properly decoded and returned in + ``.diff()`` output +* Fix: `RemoteProgress` will now strip the ', ' prefix or suffix from messages. +* API: Remote.[fetch|push|pull](...) methods now allow the ``progress`` argument to + be a callable. This saves you from creating a custom type with usually just one + implemented method. + +2.0.3 - Fixes +============= + +* Fix: bug in ``git-blame --incremental`` output parser that broken when + commit messages contained ``\r`` characters +* Fix: progress handler exceptions are not caught anymore, which would usually just hide bugs + previously. +* Fix: The `Git.execute` method will now redirect `stdout` to `devnull` if `with_stdout` is false, + which is the intended behaviour based on the parameter's documentation. + +2.0.2 - Fixes +============= + +* Fix: source package does not include \*.pyc files +* Fix: source package does include doc sources + +2.0.1 - Fixes +============= + +* Fix: remote output parser now correctly matches refs with "@" in them + +2.0.0 - Features +================ + +Please note that due to breaking changes, we have to increase the major version. + +* **IMPORTANT**: This release drops support for python 2.6, which is + officially deprecated by the python maintainers. +* **CRITICAL**: `Diff` objects created with patch output will now not carry + the --- and +++ header lines anymore. All diffs now start with the + @@ header line directly. Users that rely on the old behaviour can now + (reliably) read this information from the a_path and b_path properties + without having to parse these lines manually. +* `Commit` now has extra properties `authored_datetime` and + `committer_datetime` (to get Python datetime instances rather than + timestamps) +* `Commit.diff()` now supports diffing the root commit via + `Commit.diff(NULL_TREE)`. +* `Repo.blame()` now respects `incremental=True`, supporting incremental + blames. Incremental blames are slightly faster since they don't include + the file's contents in them. +* Fix: `Diff` objects created with patch output will now have their + `a_path` and `b_path` properties parsed out correctly. Previously, some + values may have been populated incorrectly when a file was added or + deleted. +* Fix: diff parsing issues with paths that contain "unsafe" chars, like + spaces, tabs, backslashes, etc. + +1.0.2 - Fixes +============= + +* IMPORTANT: Changed default object database of `Repo` objects to `GitCmdObjectDB`. The pure-python implementation + used previously usually fails to release its resources (i.e. file handles), which can lead to problems when working + with large repositories. +* CRITICAL: fixed incorrect `Commit` object serialization when authored or commit date had timezones which were not + divisiblej by 3600 seconds. This would happen if the timezone was something like `+0530` for instance. +* A list of all additional fixes can be found `on GitHub `__ +* CRITICAL: `Tree.cache` was removed without replacement. It is technically impossible to change individual trees and expect their serialization results to be consistent with what *git* expects. Instead, use the `IndexFile` facilities to adjust the content of the staging area, and write it out to the respective tree objects using `IndexFile.write_tree()` instead. + 1.0.1 - Fixes ============= -* A list of all issues can be found `on github `_ +* A list of all issues can be found `on GitHub `__ 1.0.0 - Notes ============= @@ -19,18 +458,18 @@ It follows the `semantic version scheme `_, and thus will not * `IndexFile.add()` will now write the index without any extension data by default. However, you may override this behaviour with the new `write_extension_data` keyword argument. - Renamed `ignore_tree_extension_data` keyword argument in `IndexFile.write(...)` to `ignore_extension_data` -* If the git command executed during `Remote.push(...)|fetch(...)` returns with an non-zero exit code and GitPython didn't +* If the git command executed during `Remote.push(...)|fetch(...)` returns with an non-zero exit code and GitPython didn't obtain any head-information, the corresponding `GitCommandError` will be raised. This may break previous code which expected - these operations to never raise. However, that behavious is undesirable as it would effectively hide the fact that there - was an error. See `this issue `_ for more information. + these operations to never raise. However, that behavious is undesirable as it would effectively hide the fact that there + was an error. See `this issue `__ for more information. * If the git executable can't be found in the PATH or at the path provided by `GIT_PYTHON_GIT_EXECUTABLE`, this is made obvious by throwing `GitCommandNotFound`, both on unix and on windows. - Those who support **GUI on windows** will now have to set `git.Git.USE_SHELL = True` to get the previous behaviour. - -* A list of all issues can be found `on github `_ - + +* A list of all issues can be found `on GitHub `__ + 0.3.6 - Features ================ @@ -44,12 +483,12 @@ It follows the `semantic version scheme `_, and thus will not * As `rev_parse` will now throw `BadName` as well as `BadObject`, client code will have to catch both exception types. * Repo.working_tree_dir now returns None if it is bare. Previously it raised AssertionError. * IndexFile.add() previously raised AssertionError when paths where used with bare repository, now it raises InvalidGitRepositoryError - -* Added `Repo.merge_base()` implementation. See the `respective issue on github `_ + +* Added `Repo.merge_base()` implementation. See the `respective issue on GitHub `__ * `[include]` sections in git configuration files are now respected * Added `GitConfigParser.rename_section()` * Added `Submodule.rename()` -* A list of all issues can be found `on github `_ +* A list of all issues can be found `on GitHub `__ 0.3.5 - Bugfixes ================ @@ -69,17 +508,17 @@ It follows the `semantic version scheme `_, and thus will not 0.3.3 ===== -* When fetching, pulling or pushing, and an error occours, it will not be reported on stdout anymore. However, if there is a fatal error, it will still result in a GitCommandError to be thrown. This goes hand in hand with improved fetch result parsing. +* When fetching, pulling or pushing, and an error occurs, it will not be reported on stdout anymore. However, if there is a fatal error, it will still result in a GitCommandError to be thrown. This goes hand in hand with improved fetch result parsing. * Code Cleanup (in preparation for python 3 support) * Applied autopep8 and cleaned up code - * Using python logging module instead of print statments to signal certain kinds of errors + * Using python logging module instead of print statements to signal certain kinds of errors 0.3.2.1 ======= * `Fix for #207 `_ -0.3.2 +0.3.2 ===== * Release of most recent version as non-RC build, just to allow pip to install the latest version right away. @@ -90,13 +529,13 @@ It follows the `semantic version scheme `_, and thus will not * **git** command wrapper * Added ``version_info`` property which returns a tuple of integers representing the installed git version. - + * Added GIT_PYTHON_GIT_EXECUTABLE environment variable, which can be used to set the desired git executable to be used. despite of what would be found in the path. - + * **Blob** Type * Added mode constants to ease the manual creation of blobs - + * **IterableList** * Added __contains__ and __delitem__ methods @@ -108,8 +547,8 @@ It follows the `semantic version scheme `_, and thus will not * Parsing of tags was improved. Previously some parts of the name could not be parsed properly. * The rev-parse pure python implementation now handles branches correctly if they look like hexadecimal sha's. * GIT_PYTHON_TRACE is now set on class level of the Git type, previously it was a module level global variable. - * GIT_PYTHON_GIT_EXECUTABLE is a class level variable as well. - + * GIT_PYTHON_GIT_EXECUTABLE is a class level variable as well. + 0.3.1 Beta 2 ============ @@ -118,7 +557,7 @@ It follows the `semantic version scheme `_, and thus will not * New types: ``RefLog`` and ``RefLogEntry`` * Reflog is maintained automatically when creating references and deleting them * Non-intrusive changes to ``SymbolicReference``, these don't require your code to change. They allow to append messages to the reflog. - + * ``abspath`` property added, similar to ``abspath`` of Object instances * ``log()`` method added * ``log_append(...)`` method added @@ -127,19 +566,19 @@ It follows the `semantic version scheme `_, and thus will not * ``set_object(...)`` method added (reflog support) * **Intrusive Changes** to ``Head`` type - + * ``create(...)`` method now supports the reflog, but will not raise ``GitCommandError`` anymore as it is a pure python implementation now. Instead, it raises ``OSError``. - + * **Intrusive Changes** to ``Repo`` type - + * ``create_head(...)`` method does not support kwargs anymore, instead it supports a logmsg parameter - + * Repo.rev_parse now supports the [ref]@{n} syntax, where *n* is the number of steps to look into the reference's past * **BugFixes** * Removed incorrect ORIG_HEAD handling - + * **Flattened directory** structure to make development more convenient. * .. note:: This alters the way projects using git-python as a submodule have to adjust their sys.path to be able to import git-python successfully. @@ -152,7 +591,7 @@ It follows the `semantic version scheme `_, and thus will not * Head Type changes * config_reader() & config_writer() methods added for access to head specific options. - * tracking_branch() & set_tracking_branch() methods addded for easy configuration of tracking branches. + * tracking_branch() & set_tracking_branch() methods added for easy configuration of tracking branches. 0.3.0 Beta 2 @@ -168,7 +607,7 @@ Renamed Modules * git.utils -> git.util * git.errors -> git.exc * git.objects.utils -> git.objects.util - + General ------- * Object instances, and everything derived from it, now use binary sha's internally. The 'sha' member was removed, in favor of the 'binsha' member. An 'hexsha' property is available for convenient conversions. They may only be initialized using their binary shas, reference names or revision specs are not allowed anymore. @@ -179,67 +618,67 @@ General * IndexFile.get_entries_key was renamed to entry_key * IndexFile.write_tree: removed missing_ok keyword, its always True now. Instead of raising GitCommandError it raises UnmergedEntriesError. This is required as the pure-python implementation doesn't support the missing_ok keyword yet. * diff.Diff.null_hex_sha renamed to NULL_HEX_SHA, to be conforming with the naming in the Object base class - + 0.2 Beta 2 =========== * Commit objects now carry the 'encoding' information of their message. It wasn't parsed previously, and defaults to UTF-8 - * Commit.create_from_tree now uses a pure-python implementation, mimicing git-commit-tree + * Commit.create_from_tree now uses a pure-python implementation, mimicking git-commit-tree 0.2 ===== General ------- -* file mode in Tree, Blob and Diff objects now is an int compatible to definintiions - in the stat module, allowing you to query whether individual user, group and other +* file mode in Tree, Blob and Diff objects now is an int compatible to definitions + in the stat module, allowing you to query whether individual user, group and other read, write and execute bits are set. * Adjusted class hierarchy to generally allow comparison and hash for Objects and Refs -* Improved Tag object which now is a Ref that may contain a tag object with additional +* Improved Tag object which now is a Ref that may contain a tag object with additional Information -* id_abbrev method has been removed as it could not assure the returned short SHA's +* id_abbrev method has been removed as it could not assure the returned short SHA's where unique * removed basename method from Objects with path's as it replicated features of os.path -* from_string and list_from_string methods are now private and were renamed to - _from_string and _list_from_string respectively. As part of the private API, they +* from_string and list_from_string methods are now private and were renamed to + _from_string and _list_from_string respectively. As part of the private API, they may change without prior notice. * Renamed all find_all methods to list_items - this method is part of the Iterable interface that also provides a more efficients and more responsive iter_items method -* All dates, like authored_date and committer_date, are stored as seconds since epoc - to consume less memory - they can be converted using time.gmtime in a more suitable +* All dates, like authored_date and committer_date, are stored as seconds since epoch + to consume less memory - they can be converted using time.gmtime in a more suitable presentation format if needed. -* Named method parameters changed on a wide scale to unify their use. Now git specific +* Named method parameters changed on a wide scale to unify their use. Now git specific terms are used everywhere, such as "Reference" ( ref ) and "Revision" ( rev ). - Prevously multiple terms where used making it harder to know which type was allowed + Previously multiple terms where used making it harder to know which type was allowed or not. * Unified diff interface to allow easy diffing between trees, trees and index, trees and working tree, index and working tree, trees and index. This closely follows the git-diff capabilities. -* Git.execute does not take the with_raw_output option anymore. It was not used +* Git.execute does not take the with_raw_output option anymore. It was not used by anyone within the project and False by default. - + Item Iteration -------------- -* Previously one would return and process multiple items as list only which can - hurt performance and memory consumption and reduce response times. - iter_items method provide an iterator that will return items on demand as parsed +* Previously one would return and process multiple items as list only which can + hurt performance and memory consumption and reduce response times. + iter_items method provide an iterator that will return items on demand as parsed from a stream. This way any amount of objects can be handled. * list_items method returns IterableList allowing to access list members by name - + objects Package ---------------- -* blob, tree, tag and commit module have been moved to new objects package. This should - not affect you though unless you explicitly imported individual objects. If you just +* blob, tree, tag and commit module have been moved to new objects package. This should + not affect you though unless you explicitly imported individual objects. If you just used the git package, names did not change. - + Blob ---- * former 'name' member renamed to path as it suits the actual data better GitCommand ----------- -* git.subcommand call scheme now prunes out None from the argument list, allowing - to be called more confortably as None can never be a valid to the git command +* git.subcommand call scheme now prunes out None from the argument list, allowing + to be called more comfortably as None can never be a valid to the git command if converted to a string. * Renamed 'git_dir' attribute to 'working_dir' which is exactly how it is used @@ -253,43 +692,43 @@ Config * The git configuration can now be read and manipulated directly from within python using the GitConfigParser * Repo.config_reader() returns a read-only parser -* Repo.config_writer() returns a read-write parser - +* Repo.config_writer() returns a read-write parser + Diff ---- * Members a a_commit and b_commit renamed to a_blob and b_blob - they are populated with Blob objects if possible * Members a_path and b_path removed as this information is kept in the blobs -* Diffs are now returned as DiffIndex allowing to more quickly find the kind of +* Diffs are now returned as DiffIndex allowing to more quickly find the kind of diffs you are interested in - + Diffing ------- -* Commit and Tree objects now support diffing natively with a common interface to - compare agains other Commits or Trees, against the working tree or against the index. +* Commit and Tree objects now support diffing natively with a common interface to + compare against other Commits or Trees, against the working tree or against the index. Index ----- * A new Index class allows to read and write index files directly, and to perform simple two and three way merges based on an arbitrary index. - -Referernces + +References ------------ * References are object that point to a Commit * SymbolicReference are a pointer to a Reference Object, which itself points to a specific Commit -* They will dynmically retrieve their object at the time of query to assure the information - is actual. Recently objects would be cached, hence ref object not be safely kept +* They will dynamically retrieve their object at the time of query to assure the information + is actual. Recently objects would be cached, hence ref object not be safely kept persistent. - + Repo ---- * Moved blame method from Blob to repo as it appeared to belong there much more. -* active_branch method now returns a Head object instead of a string with the name +* active_branch method now returns a Head object instead of a string with the name of the active branch. -* tree method now requires a Ref instance as input and defaults to the active_branche +* tree method now requires a Ref instance as input and defaults to the active_branch instead of master -* is_dirty now takes additional arguments allowing fine-grained control about what is +* is_dirty now takes additional arguments allowing fine-grained control about what is considered dirty * Removed the following methods: @@ -301,7 +740,7 @@ Repo - 'create' method which equals the 'init' method's functionality - 'diff' - it returned a mere string which still had to be parsed - 'commit_diff' - moved to Commit, Tree and Diff types respectively - + * Renamed the following methods: - commits to iter_commits to improve the performance, adjusted signature @@ -309,7 +748,7 @@ Repo - fork_bare to clone, as it was to represent general clone functionality, but implied a bare clone to be more versatile - archive_tar_gz and archive_tar and replaced by archive method with different signature - + * 'commits' method has no max-count of returned commits anymore, it now behaves like git-rev-list * The following methods and properties were added @@ -320,16 +759,16 @@ Repo - 'config_reader' method - 'config_writer' method - 'bare' property, previously it was a simple attribute that could be written - + * Renamed the following attributes - 'path' is now 'git_dir' - 'wd' is now 'working_dir' - + * Added attribute - 'working_tree_dir' which may be None in case of bare repositories - + Remote ------ * Added Remote object allowing easy access to remotes @@ -339,7 +778,7 @@ Remote Test Framework -------------- * Added support for common TestCase base class that provides additional functionality - to receive repositories tests can also write to. This way, more aspects can be + to receive repositories tests can also write to. This way, more aspects can be tested under real-world ( un-mocked ) conditions. Tree @@ -347,7 +786,7 @@ Tree * former 'name' member renamed to path as it suits the actual data better * added traverse method allowing to recursively traverse tree items * deleted blob method -* added blobs and trees properties allowing to query the respective items in the +* added blobs and trees properties allowing to query the respective items in the tree * now mimics behaviour of a read-only list instead of a dict to maintain order. * content_from_string method is now private and not part of the public API anymore @@ -363,7 +802,7 @@ General * Removed ambiguity between paths and treeishs. When calling commands that accept treeish and path arguments and there is a path with the same name as a treeish git cowardly refuses to pick one and asks for the command to use - the unambiguous syntax where '--' seperates the treeish from the paths. + the unambiguous syntax where '--' separates the treeish from the paths. * ``Repo.commits``, ``Repo.commits_between``, ``Repo.commits_since``, ``Repo.commit_count``, ``Repo.commit``, ``Commit.count`` and @@ -486,14 +925,14 @@ Git * Added support for ``stderr``, ``stdin``, and ``with_status``. -* ``git_dir`` is now optional in the constructor for ``git.Git``. Git now +* ``git_dir`` is now optional in the constructor for ``git.Git``. Git now falls back to ``os.getcwd()`` when git_dir is not specified. -* add a ``with_exceptions`` keyword argument to git commands. +* add a ``with_exceptions`` keyword argument to git commands. ``GitCommandError`` is raised when the exit status is non-zero. -* add support for a ``GIT_PYTHON_TRACE`` environment variable. - ``GIT_PYTHON_TRACE`` allows us to debug GitPython's usage of git through +* add support for a ``GIT_PYTHON_TRACE`` environment variable. + ``GIT_PYTHON_TRACE`` allows us to debug GitPython's usage of git through the use of an environment variable. Tree @@ -509,9 +948,9 @@ Repo Tree ---- -* Corrected problem with ``Tree.__div__`` not working with zero length files. - Removed ``__len__`` override and replaced with size instead. Also made size - cach properly. This is a breaking change. +* Corrected problem with ``Tree.__div__`` not working with zero length files. + Removed ``__len__`` override and replaced with size instead. Also made size + cache properly. This is a breaking change. 0.1.1 ===== diff --git a/doc/source/conf.py b/doc/source/conf.py index add686d3f..0ec64179e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -30,7 +30,7 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest'] # Add any paths that contain templates here, relative to this directory. -templates_path = ['.templates'] +templates_path = [] # The suffix of source filenames. source_suffix = '.rst' @@ -42,15 +42,16 @@ master_doc = 'index' # General information about the project. -project = u'GitPython' -copyright = u'Copyright (C) 2008, 2009 Michael Trier and contributors, 2010-2015 Sebastian Thiel' +project = 'GitPython' +copyright = 'Copyright (C) 2008, 2009 Michael Trier and contributors, 2010-2015 Sebastian Thiel' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -VERSION = open(os.path.join(os.path.dirname(__file__),"..", "..", 'VERSION')).readline().strip() +with open(os.path.join(os.path.dirname(__file__),"..", "..", 'VERSION')) as fd: + VERSION = fd.readline().strip() version = VERSION # The full version, including alpha/beta/rc tags. release = VERSION @@ -93,14 +94,10 @@ # Options for HTML output # ----------------------- +html_theme = 'sphinx_rtd_theme' html_theme_options = { } -# The style sheet to use for HTML and HTML Help pages. A file of that name -# must exist either in Sphinx' static/ path, or in one of the custom paths -# given in html_static_path. -html_style = 'default.css' - # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None @@ -120,7 +117,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['.static'] +html_static_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/doc/source/index.rst b/doc/source/index.rst index 1079c5c76..69fb573a4 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,7 +9,6 @@ GitPython Documentation :maxdepth: 2 intro - whatsnew tutorial reference roadmap diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 78d40344e..7168c91b1 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -13,17 +13,15 @@ The object database implementation is optimized for handling large quantities of Requirements ============ +* `Python`_ >= 3.5 * `Git`_ 1.7.0 or newer It should also work with older versions, but it may be that some operations involving remotes will not work as expected. * `GitDB`_ - a pure python git database implementation -* `Python Nose`_ - used for running the tests -* `Mock by Michael Foord`_ used for tests. Requires version 0.5 -.. _Git: http://git-scm.com/ -.. _Python Nose: http://code.google.com/p/python-nose/ -.. _Mock by Michael Foord: http://www.voidspace.org.uk/python/mock.html -.. _GitDB: http://pypi.python.org/pypi/gitdb +.. _Python: https://www.python.org +.. _Git: https://git-scm.com/ +.. _GitDB: https://pypi.python.org/pypi/gitdb Installing GitPython ==================== @@ -34,7 +32,7 @@ installed, just run the following from the command-line: .. sourcecode:: none - # pip install gitpython + # pip install GitPython This command will download the latest version of GitPython from the `Python Package Index `_ and install it @@ -52,9 +50,25 @@ script: .. sourcecode:: none # python setup.py install - + .. note:: In this case, you have to manually install `GitDB`_ as well. It would be recommended to use the :ref:`git source repository ` in that case. +Limitations +=========== + +Leakage of System Resources +--------------------------- + +GitPython is not suited for long-running processes (like daemons) as it tends to +leak system resources. It was written in a time where destructors (as implemented +in the `__del__` method) still ran deterministically. + +In case you still want to use it in such a context, you will want to search the +codebase for `__del__` implementations and call these yourself when you see fit. + +Another way assure proper cleanup of resources is to factor out GitPython into a +separate process which can be dropped periodically. + Getting Started =============== @@ -66,7 +80,7 @@ Getting Started API Reference ============= -An organized section of the GitPthon API is at :ref:`api_reference_toplevel`. +An organized section of the GitPython API is at :ref:`api_reference_toplevel`. .. _source-code-label: @@ -80,16 +94,16 @@ GitPython's git repo is available on GitHub, which can be browsed at: and cloned using:: $ git clone https://github.com/gitpython-developers/GitPython git-python - + Initialize all submodules to obtain the required dependencies with:: - + $ cd git-python $ git submodule update --init --recursive - -Finally verify the installation by running the `nose powered `_ unit tests:: - - $ nosetests - + +Finally verify the installation by running unit tests:: + + $ python -m unittest + Questions and Answers ===================== Please use stackoverflow for questions, and don't forget to tag it with `gitpython` to assure the right people see the question in a timely manner. @@ -98,10 +112,10 @@ http://stackoverflow.com/questions/tagged/gitpython Issue Tracker ============= -The issue tracker is hosted by github: +The issue tracker is hosted by GitHub: https://github.com/gitpython-developers/GitPython/issues - + License Information =================== GitPython is licensed under the New BSD License. See the LICENSE file for diff --git a/doc/source/reference.rst b/doc/source/reference.rst index 53fa86364..68a7f0ba4 100644 --- a/doc/source/reference.rst +++ b/doc/source/reference.rst @@ -3,6 +3,13 @@ API Reference ============= +Version +------- + +.. py:data:: git.__version__ + + Current GitPython version. + Objects.Base ------------ diff --git a/doc/source/roadmap.rst b/doc/source/roadmap.rst index f93d5e65b..a573df33a 100644 --- a/doc/source/roadmap.rst +++ b/doc/source/roadmap.rst @@ -2,7 +2,7 @@ ####### Roadmap ####### -The full list of milestones including associated tasks can be found on github: +The full list of milestones including associated tasks can be found on GitHub: https://github.com/gitpython-developers/GitPython/issues Select the respective milestone to filter the list of issues accordingly. diff --git a/doc/source/tutorial.rst b/doc/source/tutorial.rst index 2533216c9..d548f8829 100644 --- a/doc/source/tutorial.rst +++ b/doc/source/tutorial.rst @@ -10,50 +10,56 @@ GitPython Tutorial GitPython provides object model access to your git repository. This tutorial is composed of multiple sections, most of which explains a real-life usecase. -All code presented here originated from `test_docs.py `_ to assure correctness. Knowing this should also allow you to more easily run the code for your own testing purposes, all you need is a developer installation of git-python. +All code presented here originated from `test_docs.py `_ to assure correctness. Knowing this should also allow you to more easily run the code for your own testing purposes, all you need is a developer installation of git-python. Meet the Repo type ****************** The first step is to create a :class:`git.Repo ` object to represent your repository. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [1-test_init_repo_object] :end-before: # ![1-test_init_repo_object] In the above example, the directory ``self.rorepo.working_tree_dir`` equals ``/Users/mtrier/Development/git-python`` and is my working repository which contains the ``.git`` directory. You can also initialize GitPython with a *bare* repository. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [2-test_init_repo_object] :end-before: # ![2-test_init_repo_object] - + A repo object provides high-level access to your data, it allows you to create and delete heads, tags and remotes and access the configuration of the repository. - -.. literalinclude:: ../../git/test/test_docs.py + +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [3-test_init_repo_object] :end-before: # ![3-test_init_repo_object] Query the active branch, query untracked files or whether the repository data has been modified. - -.. literalinclude:: ../../git/test/test_docs.py + +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [4-test_init_repo_object] :end-before: # ![4-test_init_repo_object] - + Clone from existing repositories or initialize new empty ones. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [5-test_init_repo_object] :end-before: # ![5-test_init_repo_object] - + Archive the repository contents to a tar file. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [6-test_init_repo_object] :end-before: # ![6-test_init_repo_object] @@ -62,114 +68,129 @@ Advanced Repo Usage And of course, there is much more you can do with this type, most of the following will be explained in greater detail in specific tutorials. Don't worry if you don't understand some of these examples right away, as they may require a thorough understanding of gits inner workings. -Query relevant repository paths ... +Query relevant repository paths ... -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [7-test_init_repo_object] :end-before: # ![7-test_init_repo_object] :class:`Heads ` Heads are branches in git-speak. :class:`References ` are pointers to a specific commit or to other references. Heads and :class:`Tags ` are a kind of references. GitPython allows you to query them rather intuitively. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [8-test_init_repo_object] :end-before: # ![8-test_init_repo_object] You can also create new heads ... -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [9-test_init_repo_object] :end-before: # ![9-test_init_repo_object] -... and tags ... +... and tags ... -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [10-test_init_repo_object] :end-before: # ![10-test_init_repo_object] You can traverse down to :class:`git objects ` through references and other objects. Some objects like :class:`commits ` have additional meta-data to query. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [11-test_init_repo_object] :end-before: # ![11-test_init_repo_object] :class:`Remotes ` allow to handle fetch, pull and push operations, while providing optional real-time progress information to :class:`progress delegates `. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [12-test_init_repo_object] :end-before: # ![12-test_init_repo_object] The :class:`index ` is also called stage in git-speak. It is used to prepare new commits, and can be used to keep results of merge operations. Our index implementation allows to stream date into the index, which is useful for bare repositories that do not have a working tree. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [13-test_init_repo_object] :end-before: # ![13-test_init_repo_object] :class:`Submodules ` represent all aspects of git submodules, which allows you query all of their related information, and manipulate in various ways. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [14-test_init_repo_object] :end-before: # ![14-test_init_repo_object] - + Examining References ******************** :class:`References ` are the tips of your commit graph from which you can easily examine the history of your project. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [1-test_references_and_objects] :end-before: # ![1-test_references_and_objects] - + :class:`Tags ` are (usually immutable) references to a commit and/or a tag object. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [2-test_references_and_objects] :end-before: # ![2-test_references_and_objects] - + A :class:`symbolic reference ` is a special case of a reference as it points to another reference instead of a commit. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [3-test_references_and_objects] :end-before: # ![3-test_references_and_objects] - + Access the :class:`reflog ` easily. - -.. literalinclude:: ../../git/test/test_docs.py + +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [4-test_references_and_objects] :end-before: # ![4-test_references_and_objects] - + Modifying References ******************** You can easily create and delete :class:`reference types ` or modify where they point to. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [5-test_references_and_objects] :end-before: # ![5-test_references_and_objects] Create or delete :class:`tags ` the same way except you may not change them afterwards. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [6-test_references_and_objects] :end-before: # ![6-test_references_and_objects] - + Change the :class:`symbolic reference ` to switch branches cheaply (without adjusting the index or the working tree). -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [7-test_references_and_objects] :end-before: # ![7-test_references_and_objects] @@ -181,33 +202,37 @@ Git only knows 4 distinct object types being :class:`Blobs ` are objects that can be put into git's index. These objects are trees, blobs and submodules which additionally know about their path in the file system as well as their mode. - -.. literalinclude:: ../../git/test/test_docs.py + +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [10-test_references_and_objects] :end-before: # ![10-test_references_and_objects] - + Access :class:`blob ` data (or any object data) using streams. - -.. literalinclude:: ../../git/test/test_docs.py + +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [11-test_references_and_objects] :end-before: # ![11-test_references_and_objects] - - + + The Commit object ***************** @@ -215,38 +240,43 @@ The Commit object Obtain commits at the specified revision -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [12-test_references_and_objects] - :end-before: # ![12-test_references_and_objects] + :end-before: # ![12-test_references_and_objects] Iterate 50 commits, and if you need paging, you can specify a number of commits to skip. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [13-test_references_and_objects] - :end-before: # ![13-test_references_and_objects] + :end-before: # ![13-test_references_and_objects] A commit object carries all sorts of meta-data -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [14-test_references_and_objects] - :end-before: # ![14-test_references_and_objects] + :end-before: # ![14-test_references_and_objects] Note: date time is represented in a ``seconds since epoch`` format. Conversion to human readable form can be accomplished with the various `time module `_ methods. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [15-test_references_and_objects] - :end-before: # ![15-test_references_and_objects] + :end-before: # ![15-test_references_and_objects] You can traverse a commit's ancestry by chaining calls to ``parents`` -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [16-test_references_and_objects] - :end-before: # ![16-test_references_and_objects] + :end-before: # ![16-test_references_and_objects] The above corresponds to ``master^^^`` or ``master~3`` in git parlance. @@ -255,79 +285,89 @@ The Tree object A :class:`tree ` records pointers to the contents of a directory. Let's say you want the root tree of the latest commit on the master branch -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [17-test_references_and_objects] - :end-before: # ![17-test_references_and_objects] + :end-before: # ![17-test_references_and_objects] -Once you have a tree, you can get it's contents +Once you have a tree, you can get its contents -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [18-test_references_and_objects] - :end-before: # ![18-test_references_and_objects] + :end-before: # ![18-test_references_and_objects] It is useful to know that a tree behaves like a list with the ability to query entries by name -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [19-test_references_and_objects] - :end-before: # ![19-test_references_and_objects] + :end-before: # ![19-test_references_and_objects] There is a convenience method that allows you to get a named sub-object from a tree with a syntax similar to how paths are written in a posix system -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [20-test_references_and_objects] - :end-before: # ![20-test_references_and_objects] + :end-before: # ![20-test_references_and_objects] You can also get a commit's root tree directly from the repository -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [21-test_references_and_objects] - :end-before: # ![21-test_references_and_objects] - + :end-before: # ![21-test_references_and_objects] + As trees allow direct access to their intermediate child entries only, use the traverse method to obtain an iterator to retrieve entries recursively -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [22-test_references_and_objects] - :end-before: # ![22-test_references_and_objects] - + :end-before: # ![22-test_references_and_objects] + .. note:: If trees return Submodule objects, they will assume that they exist at the current head's commit. The tree it originated from may be rooted at another commit though, that it doesn't know. That is why the caller would have to set the submodule's owning or parent commit using the ``set_parent_commit(my_commit)`` method. - + The Index Object **************** The git index is the stage containing changes to be written with the next commit or where merges finally have to take place. You may freely access and manipulate this information using the :class:`IndexFile ` object. Modify the index with ease - -.. literalinclude:: ../../git/test/test_docs.py + +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [23-test_references_and_objects] - :end-before: # ![23-test_references_and_objects] - + :end-before: # ![23-test_references_and_objects] + Create new indices from other trees or as result of a merge. Write that result to a new index file for later inspection. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [24-test_references_and_objects] - :end-before: # ![24-test_references_and_objects] - + :end-before: # ![24-test_references_and_objects] + Handling Remotes **************** :class:`Remotes ` are used as alias for a foreign repository to ease pushing to and fetching from them -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [25-test_references_and_objects] - :end-before: # ![25-test_references_and_objects] + :end-before: # ![25-test_references_and_objects] You can easily access configuration information for a remote by accessing options as if they where attributes. The modification of remote configuration is more explicit though. - -.. literalinclude:: ../../git/test/test_docs.py + +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [26-test_references_and_objects] :end-before: # ![26-test_references_and_objects] @@ -343,7 +383,9 @@ This one sets a custom script to be executed in place of `ssh`, and can be used with repo.git.custom_environment(GIT_SSH=ssh_executable): repo.remotes.origin.fetch() -Here's an example executable that can be used in place of the `ssh_executable` above:: +Here's an example executable that can be used in place of the `ssh_executable` above: + +.. code-block:: shell #!/bin/sh ID_RSA=/var/lib/openshift/5562b947ecdd5ce939000038/app-deployments/id_rsa @@ -352,23 +394,28 @@ Here's an example executable that can be used in place of the `ssh_executable` a Please note that the script must be executable (i.e. `chomd +x script.sh`). `StrictHostKeyChecking=no` is used to avoid prompts asking to save the hosts key to `~/.ssh/known_hosts`, which happens in case you run this as daemon. You might also have a look at `Git.update_environment(...)` in case you want to setup a changed environment more permanently. - + Submodule Handling ****************** :class:`Submodules ` can be conveniently handled using the methods provided by GitPython, and as an added benefit, GitPython provides functionality which behave smarter and less error prone than its original c-git implementation, that is GitPython tries hard to keep your repository consistent when updating submodules recursively or adjusting the existing configuration. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [1-test_submodules] - :end-before: # ![1-test_submodules] + :end-before: # ![1-test_submodules] -In addition to the query functionality, you can move the submodule's repository to a different path <``move(...)``>, write its configuration <``config_writer().set_value(...).release()``>, update its working tree <``update(...)``>, and remove or add them <``remove(...)``, ``add(...)``>. +In addition to the query functionality, you can move the submodule's repository to a different path <``move(...)``>, +write its configuration <``config_writer().set_value(...).release()``>, update its working tree <``update(...)``>, +and remove or add them <``remove(...)``, ``add(...)``>. -If you obtained your submodule object by traversing a tree object which is not rooted at the head's commit, you have to inform the submodule about its actual commit to retrieve the data from by using the ``set_parent_commit(...)`` method. +If you obtained your submodule object by traversing a tree object which is not rooted at the head's commit, +you have to inform the submodule about its actual commit to retrieve the data from +by using the ``set_parent_commit(...)`` method. The special :class:`RootModule ` type allows you to treat your master repository as root of a hierarchy of submodules, which allows very convenient submodule handling. Its ``update(...)`` method is reimplemented to provide an advanced way of updating submodules as they change their values over time. The update method will track changes and make sure your working tree and submodule checkouts stay consistent, which is very useful in case submodules get deleted or added to name just two of the handled cases. -Additionally, GitPython adds functionality to track a specific branch, instead of just a commit. Supported by customized update methods, you are able to automatically update submodules to the latest revision available in the remote repository, as well as to keep track of changes and movements of these submodules. To use it, set the name of the branch you want to track to the ``submodule.$name.branch`` option of the *.gitmodules* file, and use GitPython update methods on the resulting repository with the ``to_latest_revision`` parameter turned on. In the latter case, the sha of your submodule will be ignored, instead a local tracking branch will be updated to the respective remote branch automatically, provided there are no local changes. The resulting behaviour is much like the one of svn::externals, which can be useful in times. +Additionally, GitPython adds functionality to track a specific branch, instead of just a commit. Supported by customized update methods, you are able to automatically update submodules to the latest revision available in the remote repository, as well as to keep track of changes and movements of these submodules. To use it, set the name of the branch you want to track to the ``submodule.$name.branch`` option of the *.gitmodules* file, and use GitPython update methods on the resulting repository with the ``to_latest_revision`` parameter turned on. In the latter case, the sha of your submodule will be ignored, instead a local tracking branch will be updated to the respective remote branch automatically, provided there are no local changes. The resulting behaviour is much like the one of svn::externals, which can be useful in times. Obtaining Diff Information ************************** @@ -377,45 +424,49 @@ Diffs can generally be obtained by subclasses of :class:`Diffable ` command directly. It is owned by each repository instance. -.. literalinclude:: ../../git/test/test_docs.py +.. literalinclude:: ../../test/test_docs.py :language: python + :dedent: 8 :start-after: # [31-test_references_and_objects] :end-before: # ![31-test_references_and_objects] - + The return value will by default be a string of the standard output channel produced by the command. Keyword arguments translate to short and long keyword arguments on the command-line. @@ -457,14 +510,14 @@ The type of the database determines certain performance characteristics, such as GitDB ===== The GitDB is a pure-python implementation of the git object database. It is the default database to use in GitPython 0.3. Its uses less memory when handling huge files, but will be 2 to 5 times slower when extracting large quantities small of objects from densely packed repositories:: - + repo = Repo("path/to/repo", odbt=GitDB) GitCmdObjectDB ============== The git command database uses persistent git-cat-file instances to read repository information. These operate very fast under all conditions, but will consume additional memory for the process itself. When extracting large files, memory usage will be much higher than the one of the ``GitDB``:: - + repo = Repo("path/to/repo", odbt=GitCmdObjectDB) Git Command Debugging and Customization @@ -474,9 +527,14 @@ Using environment variables, you can further adjust the behaviour of the git com * **GIT_PYTHON_TRACE** - * If set to non-0, all executed git commands will be logged using a python logger. - * if set to *full*, the executed git command and its output on stdout and stderr will be logged using a python logger. - + * If set to non-0, all executed git commands will be shown as they happen + * If set to *full*, the executed git command _and_ its entire output on stdout and stderr will be shown as they happen + + **NOTE**: All logging is outputted using a Python logger, so make sure your program is configured to show INFO-level messages. If this is not the case, try adding the following to your program:: + + import logging + logging.basicConfig(level=logging.INFO) + * **GIT_PYTHON_GIT_EXECUTABLE** * If set, it should contain the full path to the git executable, e.g. *c:\\Program Files (x86)\\Git\\bin\\git.exe* on windows or */usr/bin/git* on linux. diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst deleted file mode 100644 index e0d39b099..000000000 --- a/doc/source/whatsnew.rst +++ /dev/null @@ -1,25 +0,0 @@ - -################ -Whats New in 0.3 -################ -GitPython 0.3 is the first step in creating a hybrid which uses a pure python implementations for all simple git features which can be implemented without significant performance penalties. Everything else is still performed using the git command, which is nicely integrated and easy to use. - -Its biggest strength, being the support for all git features through the git command itself, is a weakness as well considering the possibly vast amount of times the git command is being started up. Depending on the actual command being performed, the git repository will be initialized on many of these invocations, causing additional overhead for possibly tiny operations. - -Keeping as many major operations in the python world will result in improved caching benefits as certain data structures just have to be initialized once and can be reused multiple times. This mode of operation may improve performance when altering the git database on a low level, and is clearly beneficial on operating systems where command invocations are very slow. - -**************** -Object Databases -**************** -An object database provides a simple interface to query object information or to write new object data. Objects are generally identified by their 20 byte binary sha1 value during query. - -GitPython uses the ``gitdb`` project to provide a pure-python implementation of the git database, which includes reading and writing loose objects, reading pack files and handling alternate repositories. - -The great thing about this is that ``Repo`` objects can use any object database, hence it easily supports different implementations with different performance characteristics. If you are thinking in extremes, you can implement your own database representation, which may be more efficient for what you want to do specifically, like handling big files more efficiently. - -************************ -Reduced Memory Footprint -************************ -Objects, such as commits, tags, trees and blobs now use 20 byte sha1 signatures internally, reducing their memory demands by 20 bytes per object, allowing you to keep more objects in memory at the same time. - -The internal caches of tree objects were improved to use less memory as well. diff --git a/dockernose.sh b/dockernose.sh new file mode 100755 index 000000000..c9227118a --- /dev/null +++ b/dockernose.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex +if [ -z "${PYVER}" ]; then + PYVER=py37 +fi + +# remember to use "-s" if you inject pdb.set_trace() as this disables nosetests capture of streams + +tox -e ${PYVER} --notest +PYTHONPATH=/src/.tox/${PYVER}/lib/python*/site-packages /src/.tox/${PYVER}/bin/nosetests --pdb $* diff --git a/git/__init__.py b/git/__init__.py index e8dae2723..ae9254a26 100644 --- a/git/__init__.py +++ b/git/__init__.py @@ -4,52 +4,87 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php # flake8: noqa - +#@PydevCodeAnalysisIgnore +from git.exc import * # @NoMove @IgnorePep8 +import inspect import os import sys -import inspect +import os.path as osp + +from typing import Optional +from git.types import PathLike __version__ = 'git' #{ Initialization -def _init_externals(): +def _init_externals() -> None: """Initialize external projects by putting them into the path""" - if __version__ == 'git': - sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'ext', 'gitdb')) + if __version__ == 'git' and 'PYOXIDIZER' not in os.environ: + sys.path.insert(1, osp.join(osp.dirname(__file__), 'ext', 'gitdb')) try: import gitdb - except ImportError: - raise ImportError("'gitdb' could not be found in your PYTHONPATH") + except ImportError as e: + raise ImportError("'gitdb' could not be found in your PYTHONPATH") from e # END verify import #} END initialization + ################# _init_externals() ################# #{ Imports -from git.config import GitConfigParser -from git.objects import * -from git.refs import * -from git.diff import * -from git.exc import * -from git.db import * -from git.cmd import Git -from git.repo import Repo -from git.remote import * -from git.index import * -from git.util import ( - LockFile, - BlockingLockFile, - Stats, - Actor -) +try: + from git.config import GitConfigParser # @NoMove @IgnorePep8 + from git.objects import * # @NoMove @IgnorePep8 + from git.refs import * # @NoMove @IgnorePep8 + from git.diff import * # @NoMove @IgnorePep8 + from git.db import * # @NoMove @IgnorePep8 + from git.cmd import Git # @NoMove @IgnorePep8 + from git.repo import Repo # @NoMove @IgnorePep8 + from git.remote import * # @NoMove @IgnorePep8 + from git.index import * # @NoMove @IgnorePep8 + from git.util import ( # @NoMove @IgnorePep8 + LockFile, + BlockingLockFile, + Stats, + Actor, + rmtree, + ) +except GitError as exc: + raise ImportError('%s: %s' % (exc.__class__.__name__, exc)) from exc #} END imports __all__ = [name for name, obj in locals().items() if not (name.startswith('_') or inspect.ismodule(obj))] + + +#{ Initialize git executable path +GIT_OK = None + + +def refresh(path: Optional[PathLike] = None) -> None: + """Convenience method for setting the git executable path.""" + global GIT_OK + GIT_OK = False + + if not Git.refresh(path=path): + return + if not FetchInfo.refresh(): + return + + GIT_OK = True +#} END initialize git executable path + + +################# +try: + refresh() +except Exception as exc: + raise ImportError('Failed to initialize: {0}'.format(exc)) from exc +################# diff --git a/git/cmd.py b/git/cmd.py index 31865d090..40e32e370 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -4,59 +4,51 @@ # This module is part of GitPython and is released under # the BSD License: http://www.opensource.org/licenses/bsd-license.php -import os -import os.path -import sys -import select -import logging -import threading -import errno -import mmap - from contextlib import contextmanager +import io +import logging +import os +import signal from subprocess import ( call, Popen, PIPE ) +import subprocess +import sys +import threading +from collections import OrderedDict +from textwrap import dedent +import warnings - -from .util import ( - LazyMixin, - stream_copy, - WaitGroup +from git.compat import ( + defenc, + force_bytes, + safe_decode, + is_posix, + is_win, ) +from git.exc import CommandError +from git.util import is_cygwin_git, cygpath, expand_path, remove_password_if_present + from .exc import ( GitCommandError, GitCommandNotFound ) -from git.compat import ( - string_types, - defenc, - PY3, - bchr, - # just to satisfy flake8 on py3 - unicode +from .util import ( + LazyMixin, + stream_copy, ) -execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output', +execute_kwargs = {'istream', 'with_extended_output', 'with_exceptions', 'as_process', 'stdout_as_string', - 'output_stream', 'with_stdout') + 'output_stream', 'with_stdout', 'kill_after_timeout', + 'universal_newlines', 'shell', 'env', 'max_chunk_size'} -log = logging.getLogger('git.cmd') +log = logging.getLogger(__name__) log.addHandler(logging.NullHandler()) -__all__ = ('Git', ) - -if sys.platform != 'win32': - WindowsError = OSError - -if PY3: - _bchr = bchr -else: - def _bchr(c): - return c -# get custom byte character handling +__all__ = ('Git',) # ============================================================================== @@ -65,158 +57,91 @@ def _bchr(c): # Documentation ## @{ -def handle_process_output(process, stdout_handler, stderr_handler, finalizer): - """Registers for notifications to lean that process output is ready to read, and dispatches lines to - the respective line handlers. We are able to handle carriage returns in case progress is sent by that - mean. For performance reasons, we only apply this to stderr. +def handle_process_output(process, stdout_handler, stderr_handler, + finalizer=None, decode_streams=True): + """Registers for notifications to learn that process output is ready to read, and dispatches lines to + the respective line handlers. This function returns once the finalizer returns + :return: result of finalizer :param process: subprocess.Popen instance :param stdout_handler: f(stdout_line_string), or None - :param stderr_hanlder: f(stderr_line_string), or None - :param finalizer: f(proc) - wait for proc to finish""" - fdmap = {process.stdout.fileno(): (stdout_handler, [b'']), - process.stderr.fileno(): (stderr_handler, [b''])} - - def _parse_lines_from_buffer(buf): - line = b'' - bi = 0 - lb = len(buf) - while bi < lb: - char = _bchr(buf[bi]) - bi += 1 - - if char in (b'\r', b'\n') and line: - yield bi, line - line = b'' - else: - line += char - # END process parsed line - # END while file is not done reading - # end - - def _read_lines_from_fno(fno, last_buf_list): - buf = os.read(fno, mmap.PAGESIZE) - buf = last_buf_list[0] + buf - - bi = 0 - for bi, line in _parse_lines_from_buffer(buf): - yield line - # for each line to parse from the buffer - - # keep remainder - last_buf_list[0] = buf[bi:] - - def _dispatch_single_line(line, handler): - line = line.decode(defenc) - if line and handler: - try: - handler(line) - except Exception: - # Keep reading, have to pump the lines empty nontheless - log.error("Line handler exception on line: %s", line, exc_info=True) - # end - # end dispatch helper - # end single line helper - - def _dispatch_lines(fno, handler, buf_list): - lc = 0 - for line in _read_lines_from_fno(fno, buf_list): - _dispatch_single_line(line, handler) - lc += 1 - # for each line - return lc - # end - - def _deplete_buffer(fno, handler, buf_list, wg=None): - lc = 0 - while True: - line_count = _dispatch_lines(fno, handler, buf_list) - lc += line_count - if line_count == 0: - break - # end deplete buffer - - if buf_list[0]: - _dispatch_single_line(buf_list[0], handler) - lc += 1 - # end - - if wg: - wg.done() + :param stderr_handler: f(stderr_line_string), or None + :param finalizer: f(proc) - wait for proc to finish + :param decode_streams: + Assume stdout/stderr streams are binary and decode them before pushing \ + their contents to handlers. + Set it to False if `universal_newline == True` (then streams are in text-mode) + or if decoding must happen later (i.e. for Diffs). + """ + # Use 2 "pump" threads and wait for both to finish. + def pump_stream(cmdline, name, stream, is_decode, handler): + try: + for line in stream: + if handler: + if is_decode: + line = line.decode(defenc) + handler(line) + except Exception as ex: + log.error("Pumping %r of cmd(%s) failed due to: %r", name, remove_password_if_present(cmdline), ex) + raise CommandError(['<%s-pump>' % name] + remove_password_if_present(cmdline), ex) from ex + finally: + stream.close() - return lc - # end + cmdline = getattr(process, 'args', '') # PY3+ only + if not isinstance(cmdline, (tuple, list)): + cmdline = cmdline.split() - if hasattr(select, 'poll'): - # poll is preferred, as select is limited to file handles up to 1024 ... . This could otherwise be - # an issue for us, as it matters how many handles our own process has - poll = select.poll() - READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR - CLOSED = select.POLLHUP | select.POLLERR + pumps = [] + if process.stdout: + pumps.append(('stdout', process.stdout, stdout_handler)) + if process.stderr: + pumps.append(('stderr', process.stderr, stderr_handler)) - poll.register(process.stdout, READ_ONLY) - poll.register(process.stderr, READ_ONLY) + threads = [] - closed_streams = set() - while True: - # no timeout + for name, stream, handler in pumps: + t = threading.Thread(target=pump_stream, + args=(cmdline, name, stream, decode_streams, handler)) + t.setDaemon(True) + t.start() + threads.append(t) - try: - poll_result = poll.poll() - except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise - # end handle poll exception - - for fd, result in poll_result: - if result & CLOSED: - closed_streams.add(fd) - else: - _dispatch_lines(fd, *fdmap[fd]) - # end handle closed stream - # end for each poll-result tuple - - if len(closed_streams) == len(fdmap): - break - # end its all done - # end endless loop - - # Depelete all remaining buffers - for fno, (handler, buf_list) in fdmap.items(): - _deplete_buffer(fno, handler, buf_list) - # end for each file handle - - for fno in fdmap.keys(): - poll.unregister(fno) - # end don't forget to unregister ! - else: - # Oh ... probably we are on windows. select.select() can only handle sockets, we have files - # The only reliable way to do this now is to use threads and wait for both to finish - # Since the finalizer is expected to wait, we don't have to introduce our own wait primitive - # NO: It's not enough unfortunately, and we will have to sync the threads - wg = WaitGroup() - for fno, (handler, buf_list) in fdmap.items(): - wg.add(1) - t = threading.Thread(target=lambda: _deplete_buffer(fno, handler, buf_list, wg)) - t.start() - # end - # NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's - # actually started, which could make the wait() call to just return because the thread is not yet - # active - wg.wait() - # end + ## FIXME: Why Join?? Will block if `stdin` needs feeding... + # + for t in threads: + t.join() - return finalizer(process) + if finalizer: + return finalizer(process) def dashify(string): return string.replace('_', '-') + +def slots_to_dict(self, exclude=()): + return {s: getattr(self, s) for s in self.__slots__ if s not in exclude} + + +def dict_to_slots_and__excluded_are_none(self, d, excluded=()): + for k, v in d.items(): + setattr(self, k, v) + for k in excluded: + setattr(self, k, None) + ## -- End Utilities -- @} +# value of Windows process creation flag taken from MSDN +CREATE_NO_WINDOW = 0x08000000 + +## CREATE_NEW_PROCESS_GROUP is needed to allow killing it afterwards, +# see https://docs.python.org/3/library/subprocess.html#subprocess.Popen.send_signal +PROC_CREATIONFLAGS = (CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP + if is_win else 0) + + class Git(LazyMixin): """ @@ -234,36 +159,184 @@ class Git(LazyMixin): Set its value to 'full' to see details about the returned values. """ __slots__ = ("_working_dir", "cat_file_all", "cat_file_header", "_version_info", - "_git_options", "_environment") + "_git_options", "_persistent_git_options", "_environment") + + _excluded_ = ('cat_file_all', 'cat_file_header', '_version_info') + + def __getstate__(self): + return slots_to_dict(self, exclude=self._excluded_) + + def __setstate__(self, d): + dict_to_slots_and__excluded_are_none(self, d, excluded=self._excluded_) # CONFIGURATION - # The size in bytes read from stdout when copying git's output to another stream - max_chunk_size = 1024 * 64 git_exec_name = "git" # default that should work on linux and windows - git_exec_name_win = "git.cmd" # alternate command name, windows only # Enables debugging of GitPython's git commands GIT_PYTHON_TRACE = os.environ.get("GIT_PYTHON_TRACE", False) - # Provide the full path to the git executable. Otherwise it assumes git is in the path - _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" - GIT_PYTHON_GIT_EXECUTABLE = os.environ.get(_git_exec_env_var, git_exec_name) - # If True, a shell will be used when executing git commands. - # This should only be desirable on windows, see https://github.com/gitpython-developers/GitPython/pull/126 - # for more information + # This should only be desirable on Windows, see https://github.com/gitpython-developers/GitPython/pull/126 + # and check `git/test_repo.py:TestRepo.test_untracked_files()` TC for an example where it is required. # Override this value using `Git.USE_SHELL = True` USE_SHELL = False - class AutoInterrupt(object): + # Provide the full path to the git executable. Otherwise it assumes git is in the path + _git_exec_env_var = "GIT_PYTHON_GIT_EXECUTABLE" + _refresh_env_var = "GIT_PYTHON_REFRESH" + GIT_PYTHON_GIT_EXECUTABLE = None + # note that the git executable is actually found during the refresh step in + # the top level __init__ + + @classmethod + def refresh(cls, path=None): + """This gets called by the refresh function (see the top level + __init__). + """ + # discern which path to refresh with + if path is not None: + new_git = os.path.expanduser(path) + new_git = os.path.abspath(new_git) + else: + new_git = os.environ.get(cls._git_exec_env_var, cls.git_exec_name) + + # keep track of the old and new git executable path + old_git = cls.GIT_PYTHON_GIT_EXECUTABLE + cls.GIT_PYTHON_GIT_EXECUTABLE = new_git + + # test if the new git executable path is valid + + # - a GitCommandNotFound error is spawned by ourselves + # - a PermissionError is spawned if the git executable provided + # cannot be executed for whatever reason + + has_git = False + try: + cls().version() + has_git = True + except (GitCommandNotFound, PermissionError): + pass + + # warn or raise exception if test failed + if not has_git: + err = dedent("""\ + Bad git executable. + The git executable must be specified in one of the following ways: + - be included in your $PATH + - be set via $%s + - explicitly set via git.refresh() + """) % cls._git_exec_env_var + + # revert to whatever the old_git was + cls.GIT_PYTHON_GIT_EXECUTABLE = old_git + + if old_git is None: + # on the first refresh (when GIT_PYTHON_GIT_EXECUTABLE is + # None) we only are quiet, warn, or error depending on the + # GIT_PYTHON_REFRESH value + + # determine what the user wants to happen during the initial + # refresh we expect GIT_PYTHON_REFRESH to either be unset or + # be one of the following values: + # 0|q|quiet|s|silence + # 1|w|warn|warning + # 2|r|raise|e|error + + mode = os.environ.get(cls._refresh_env_var, "raise").lower() + + quiet = ["quiet", "q", "silence", "s", "none", "n", "0"] + warn = ["warn", "w", "warning", "1"] + error = ["error", "e", "raise", "r", "2"] + + if mode in quiet: + pass + elif mode in warn or mode in error: + err = dedent("""\ + %s + All git commands will error until this is rectified. + + This initial warning can be silenced or aggravated in the future by setting the + $%s environment variable. Use one of the following values: + - %s: for no warning or exception + - %s: for a printed warning + - %s: for a raised exception + + Example: + export %s=%s + """) % ( + err, + cls._refresh_env_var, + "|".join(quiet), + "|".join(warn), + "|".join(error), + cls._refresh_env_var, + quiet[0]) + + if mode in warn: + print("WARNING: %s" % err) + else: + raise ImportError(err) + else: + err = dedent("""\ + %s environment variable has been set but it has been set with an invalid value. + + Use only the following values: + - %s: for no warning or exception + - %s: for a printed warning + - %s: for a raised exception + """) % ( + cls._refresh_env_var, + "|".join(quiet), + "|".join(warn), + "|".join(error)) + raise ImportError(err) + + # we get here if this was the init refresh and the refresh mode + # was not error, go ahead and set the GIT_PYTHON_GIT_EXECUTABLE + # such that we discern the difference between a first import + # and a second import + cls.GIT_PYTHON_GIT_EXECUTABLE = cls.git_exec_name + else: + # after the first refresh (when GIT_PYTHON_GIT_EXECUTABLE + # is no longer None) we raise an exception + raise GitCommandNotFound("git", err) + + return has_git + + @classmethod + def is_cygwin(cls): + return is_cygwin_git(cls.GIT_PYTHON_GIT_EXECUTABLE) + + @classmethod + def polish_url(https://github.com/cls,%20url,%20is_cygwin=None): + if is_cygwin is None: + is_cygwin = cls.is_cygwin() + + if is_cygwin: + url = cygpath(url) + else: + """Remove any backslahes from urls to be written in config files. + Windows might create config-files containing paths with backslashed, + but git stops liking them as it will escape the backslashes. + Hence we undo the escaping just to be sure. + """ + url = os.path.expandvars(url) + if url.startswith('~'): + url = os.path.expanduser(url) + url = url.replace("\\\\", "\\").replace("\\", "/") + + return url + + class AutoInterrupt(object): """Kill/Interrupt the stored process instance once this instance goes out of scope. It is used to prevent processes piling up in case iterators stop reading. Besides all attributes are wired through to the contained process object. The wait method was overridden to perform automatic status code checking and possibly raise.""" + __slots__ = ("proc", "args") def __init__(self, proc, args): @@ -278,41 +351,62 @@ def __del__(self): self.proc = None if proc.stdin: proc.stdin.close() - proc.stdout.close() - proc.stderr.close() + if proc.stdout: + proc.stdout.close() + if proc.stderr: + proc.stderr.close() # did the process finish already so we have a return code ? - if proc.poll() is not None: - return + try: + if proc.poll() is not None: + return + except OSError as ex: + log.info("Ignored error after process had died: %r", ex) # can be that nothing really exists anymore ... - if os is None: + if os is None or getattr(os, 'kill', None) is None: return # try to kill it try: - os.kill(proc.pid, 2) # interrupt signal + proc.terminate() proc.wait() # ensure process goes away - except (OSError, WindowsError): - pass # ignore error when process already died + except OSError as ex: + log.info("Ignored error after process had died: %r", ex) except AttributeError: # try windows # for some reason, providing None for stdout/stderr still prints something. This is why # we simply use the shell and redirect to nul. Its slower than CreateProcess, question # is whether we really want to see all these messages. Its annoying no matter what. - call(("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(proc.pid)), shell=True) + if is_win: + call(("TASKKILL /F /T /PID %s 2>nul 1>nul" % str(proc.pid)), shell=True) # END exception handling def __getattr__(self, attr): return getattr(self.proc, attr) - def wait(self): + def wait(self, stderr=b''): # TODO: Bad choice to mimic `proc.wait()` but with different args. """Wait for the process and return its status code. + :param stderr: Previously read value of stderr, in case stderr is already closed. + :warn: may deadlock if output or error pipes are used and not handled separately. :raise GitCommandError: if the return status is not 0""" + if stderr is None: + stderr = b'' + stderr = force_bytes(data=stderr, encoding='utf-8') + status = self.proc.wait() + + def read_all_from_possibly_closed_stream(stream): + try: + return stderr + force_bytes(stream.read()) + except ValueError: + return stderr or b'' + if status != 0: - raise GitCommandError(self.args, status, self.proc.stderr.read()) + errstr = read_all_from_possibly_closed_stream(self.proc.stderr) + log.debug('AutoInterrupt wait stderr: %r' % (errstr,)) + raise GitCommandError(remove_password_if_present(self.args), status, errstr) # END status handling return status # END auto interrupt @@ -342,7 +436,7 @@ def __init__(self, size, stream): def read(self, size=-1): bytes_left = self._size - self._nbr if bytes_left == 0: - return '' + return b'' if size > -1: # assure we don't try to read past our limit size = min(bytes_left, size) @@ -361,7 +455,7 @@ def read(self, size=-1): def readline(self, size=-1): if self._nbr == self._size: - return '' + return b'' # clamp size to lowest allowed value bytes_left = self._size - self._nbr @@ -383,10 +477,10 @@ def readline(self, size=-1): def readlines(self, size=-1): if self._nbr == self._size: - return list() + return [] # leave all additional logic to our readline method, we just check the size - out = list() + out = [] nbr = 0 while True: line = self.readline() @@ -401,13 +495,18 @@ def readlines(self, size=-1): # END readline loop return out + # skipcq: PYL-E0301 def __iter__(self): return self + def __next__(self): + return self.next() + def next(self): line = self.readline() if not line: raise StopIteration + return line def __del__(self): @@ -427,8 +526,9 @@ def __init__(self, working_dir=None): It is meant to be the working tree directory if available, or the .git directory in case of bare repositories.""" super(Git, self).__init__() - self._working_dir = working_dir + self._working_dir = expand_path(working_dir) self._git_options = () + self._persistent_git_options = [] # Extra environment variables to pass to git commands self._environment = {} @@ -445,9 +545,23 @@ def __getattr__(self, name): return LazyMixin.__getattr__(self, name) return lambda *args, **kwargs: self._call_process(name, *args, **kwargs) + def set_persistent_git_options(self, **kwargs): + """Specify command line options to the git executable + for subsequent subcommand calls + + :param kwargs: + is a dict of keyword arguments. + these arguments are passed as in _call_process + but will be passed to the git command rather than + the subcommand. + """ + + self._persistent_git_options = self.transform_kwargs( + split_single_char_options=True, **kwargs) + def _set_cache_(self, attr): if attr == '_version_info': - # We only use the first 4 numbers, as everthing else could be strings in fact (on windows) + # We only use the first 4 numbers, as everything else could be strings in fact (on windows) version_numbers = self._call_process('version').split(' ')[2] self._version_info = tuple(int(n) for n in version_numbers.split('.')[:4] if n.isdigit()) else: @@ -469,13 +583,17 @@ def version_info(self): def execute(self, command, istream=None, - with_keep_cwd=False, with_extended_output=False, with_exceptions=True, as_process=False, output_stream=None, stdout_as_string=True, + kill_after_timeout=None, with_stdout=True, + universal_newlines=False, + shell=None, + env=None, + max_chunk_size=io.DEFAULT_BUFFER_SIZE, **subprocess_kwargs ): """Handles executing the command on the shell and consumes and returns @@ -489,11 +607,6 @@ def execute(self, command, :param istream: Standard input filehandle passed to subprocess.Popen. - :param with_keep_cwd: - Whether to use the current working directory from os.getcwd(). - The cmd otherwise uses its own working_dir that it has been initialized - with if possible. - :param with_extended_output: Whether to return a (status, stdout, stderr) tuple. @@ -524,18 +637,41 @@ def execute(self, command, decoded into a string using the default encoding (usually utf-8). The latter can fail, if the output contains binary data. + :param env: + A dictionary of environment variables to be passed to `subprocess.Popen`. + + :param max_chunk_size: + Maximum number of bytes in one chunk of data passed to the output_stream in + one invocation of write() method. If the given number is not positive then + the default value is used. + :param subprocess_kwargs: Keyword arguments to be passed to subprocess.Popen. Please note that some of the valid kwargs are already set by this method, the ones you specify may not be the same ones. :param with_stdout: If True, default True, we open stdout on the created process + :param universal_newlines: + if True, pipes will be opened as text, and lines are split at + all known line endings. + :param shell: + Whether to invoke commands through a shell (see `Popen(..., shell=True)`). + It overrides :attr:`USE_SHELL` if it is not `None`. + :param kill_after_timeout: + To specify a timeout in seconds for the git command, after which the process + should be killed. This will have no effect if as_process is set to True. It is + set to None by default and will let the process run until the timeout is + explicitly specified. This feature is not supported on Windows. It's also worth + noting that kill_after_timeout uses SIGKILL, which can have negative side + effects on a repository. For example, stale locks in case of git gc could + render the repository incapable of accepting changes until the lock is manually + removed. :return: * str(output) if extended_output = False (Default) * tuple(int(status), str(stdout), str(stderr)) if extended_output = True - if ouput_stream is True, the stdout value will be your output stream: + if output_stream is True, the stdout value will be your output stream: * output_stream if extended_output = False * tuple(int(status), output_stream, str(stderr)) if extended_output = True @@ -547,16 +683,16 @@ def execute(self, command, :note: If you add additional keyword arguments to the signature of this method, you must update the execute_kwargs tuple housed in this module.""" + # Remove password for the command if present + redacted_command = remove_password_if_present(command) if self.GIT_PYTHON_TRACE and (self.GIT_PYTHON_TRACE != 'full' or as_process): - log.info(' '.join(command)) + log.info(' '.join(redacted_command)) # Allow the user to have the command executed in their working dir. - if with_keep_cwd or self._working_dir is None: - cwd = os.getcwd() - else: - cwd = self._working_dir + cwd = self._working_dir or os.getcwd() # Start the process + inline_env = env env = os.environ.copy() # Attempt to force all output to plain ascii english, which is what some parsing code # may expect. @@ -565,52 +701,109 @@ def execute(self, command, env["LANGUAGE"] = "C" env["LC_ALL"] = "C" env.update(self._environment) + if inline_env is not None: + env.update(inline_env) - if sys.platform == 'win32': - cmd_not_found_exception = WindowsError + if is_win: + cmd_not_found_exception = OSError + if kill_after_timeout: + raise GitCommandError(redacted_command, '"kill_after_timeout" feature is not supported on Windows.') else: if sys.version_info[0] > 2: - cmd_not_found_exception = FileNotFoundError # NOQA # this is defined, but flake8 doesn't know + cmd_not_found_exception = FileNotFoundError # NOQA # exists, flake8 unknown @UndefinedVariable else: cmd_not_found_exception = OSError # end handle + stdout_sink = (PIPE + if with_stdout + else getattr(subprocess, 'DEVNULL', None) or open(os.devnull, 'wb')) + istream_ok = "None" + if istream: + istream_ok = "" + log.debug("Popen(%s, cwd=%s, universal_newlines=%s, shell=%s, istream=%s)", + redacted_command, cwd, universal_newlines, shell, istream_ok) try: proc = Popen(command, env=env, cwd=cwd, + bufsize=-1, stdin=istream, stderr=PIPE, - stdout=with_stdout and PIPE or None, - shell=self.USE_SHELL, - close_fds=(os.name == 'posix'), # unsupported on windows + stdout=stdout_sink, + shell=shell is not None and shell or self.USE_SHELL, + close_fds=is_posix, # unsupported on windows + universal_newlines=universal_newlines, + creationflags=PROC_CREATIONFLAGS, **subprocess_kwargs ) except cmd_not_found_exception as err: - raise GitCommandNotFound(str(err)) + raise GitCommandNotFound(redacted_command, err) from err if as_process: return self.AutoInterrupt(proc, command) + def _kill_process(pid): + """ Callback method to kill a process. """ + p = Popen(['ps', '--ppid', str(pid)], stdout=PIPE, + creationflags=PROC_CREATIONFLAGS) + child_pids = [] + for line in p.stdout: + if len(line.split()) > 0: + local_pid = (line.split())[0] + if local_pid.isdigit(): + child_pids.append(int(local_pid)) + try: + # Windows does not have SIGKILL, so use SIGTERM instead + sig = getattr(signal, 'SIGKILL', signal.SIGTERM) + os.kill(pid, sig) + for child_pid in child_pids: + try: + os.kill(child_pid, sig) + except OSError: + pass + kill_check.set() # tell the main routine that the process was killed + except OSError: + # It is possible that the process gets completed in the duration after timeout + # happens and before we try to kill the process. + pass + return + # end + + if kill_after_timeout: + kill_check = threading.Event() + watchdog = threading.Timer(kill_after_timeout, _kill_process, args=(proc.pid,)) + # Wait for the process to return status = 0 stdout_value = b'' stderr_value = b'' + newline = "\n" if universal_newlines else b"\n" try: if output_stream is None: + if kill_after_timeout: + watchdog.start() stdout_value, stderr_value = proc.communicate() + if kill_after_timeout: + watchdog.cancel() + if kill_check.isSet(): + stderr_value = ('Timeout: the command "%s" did not complete in %d ' + 'secs.' % (" ".join(redacted_command), kill_after_timeout)) + if not universal_newlines: + stderr_value = stderr_value.encode(defenc) # strip trailing "\n" - if stdout_value.endswith(b"\n"): + if stdout_value.endswith(newline): stdout_value = stdout_value[:-1] - if stderr_value.endswith(b"\n"): + if stderr_value.endswith(newline): stderr_value = stderr_value[:-1] status = proc.returncode else: - stream_copy(proc.stdout, output_stream, self.max_chunk_size) - stdout_value = output_stream + max_chunk_size = max_chunk_size if max_chunk_size and max_chunk_size > 0 else io.DEFAULT_BUFFER_SIZE + stream_copy(proc.stdout, output_stream, max_chunk_size) + stdout_value = proc.stdout.read() stderr_value = proc.stderr.read() # strip trailing "\n" - if stderr_value.endswith(b"\n"): + if stderr_value.endswith(newline): stderr_value = stderr_value[:-1] status = proc.wait() # END stdout handling @@ -619,15 +812,15 @@ def execute(self, command, proc.stderr.close() if self.GIT_PYTHON_TRACE == 'full': - cmdstr = " ".join(command) + cmdstr = " ".join(redacted_command) def as_text(stdout_value): - return not output_stream and stdout_value.decode(defenc) or '' + return not output_stream and safe_decode(stdout_value) or '' # end if stderr_value: log.info("%s -> %d; stdout: '%s'; stderr: '%s'", - cmdstr, status, as_text(stdout_value), stderr_value.decode(defenc)) + cmdstr, status, as_text(stdout_value), safe_decode(stderr_value)) elif stdout_value: log.info("%s -> %d; stdout: '%s'", cmdstr, status, as_text(stdout_value)) else: @@ -635,17 +828,14 @@ def as_text(stdout_value): # END handle debug printing if with_exceptions and status != 0: - if with_extended_output: - raise GitCommandError(command, status, stderr_value, stdout_value) - else: - raise GitCommandError(command, status, stderr_value) + raise GitCommandError(redacted_command, status, stderr_value, stdout_value) if isinstance(stdout_value, bytes) and stdout_as_string: # could also be output_stream - stdout_value = stdout_value.decode(defenc) + stdout_value = safe_decode(stdout_value) # Allow access to the command's status code if with_extended_output: - return (status, stdout_value, stderr_value.decode(defenc)) + return (status, stdout_value, safe_decode(stderr_value)) else: return stdout_value @@ -670,10 +860,7 @@ def update_environment(self, **kwargs): for key, value in kwargs.items(): # set value if it is None if value is not None: - if key in self._environment: - old_env[key] = self._environment[key] - else: - old_env[key] = None + old_env[key] = self._environment.get(key) self._environment[key] = value # remove key from environment if its value is None elif key in self._environment: @@ -700,40 +887,49 @@ def custom_environment(self, **kwargs): finally: self.update_environment(**old_env) - def transform_kwargs(self, split_single_char_options=False, **kwargs): + def transform_kwarg(self, name, value, split_single_char_options): + if len(name) == 1: + if value is True: + return ["-%s" % name] + elif value not in (False, None): + if split_single_char_options: + return ["-%s" % name, "%s" % value] + else: + return ["-%s%s" % (name, value)] + else: + if value is True: + return ["--%s" % dashify(name)] + elif value is not False and value is not None: + return ["--%s=%s" % (dashify(name), value)] + return [] + + def transform_kwargs(self, split_single_char_options=True, **kwargs): """Transforms Python style kwargs into git command line options.""" - args = list() + # Python 3.6 preserves the order of kwargs and thus has a stable + # order. For older versions sort the kwargs by the key to get a stable + # order. + if sys.version_info[:2] < (3, 6): + kwargs = OrderedDict(sorted(kwargs.items(), key=lambda x: x[0])) + warnings.warn("Python 3.5 support is deprecated and will be removed 2021-09-05.\n" + + "It does not preserve the order for key-word arguments and enforce lexical sorting instead.") + args = [] for k, v in kwargs.items(): - if len(k) == 1: - if v is True: - args.append("-%s" % k) - elif type(v) is not bool: - if split_single_char_options: - args.extend(["-%s" % k, "%s" % v]) - else: - args.append("-%s%s" % (k, v)) + if isinstance(v, (list, tuple)): + for value in v: + args += self.transform_kwarg(k, value, split_single_char_options) else: - if v is True: - args.append("--%s" % dashify(k)) - elif type(v) is not bool: - args.append("--%s=%s" % (dashify(k), v)) + args += self.transform_kwarg(k, v, split_single_char_options) return args @classmethod def __unpack_args(cls, arg_list): if not isinstance(arg_list, (list, tuple)): - # This is just required for unicode conversion, as subprocess can't handle it - # However, in any other case, passing strings (usually utf-8 encoded) is totally fine - if not PY3 and isinstance(arg_list, unicode): - return [arg_list.encode(defenc)] return [str(arg_list)] - outlist = list() + outlist = [] for arg in arg_list: if isinstance(arg_list, (list, tuple)): outlist.extend(cls.__unpack_args(arg)) - elif not PY3 and isinstance(arg_list, unicode): - outlist.append(arg_list.encode(defenc)) # END recursion else: outlist.append(str(arg)) @@ -770,27 +966,30 @@ def _call_process(self, method, *args, **kwargs): is realized as non-existent :param kwargs: - is a dict of keyword arguments. - This function accepts the same optional keyword arguments - as execute(). + It contains key-values for the following: + - the :meth:`execute()` kwds, as listed in :var:`execute_kwargs`; + - "command options" to be converted by :meth:`transform_kwargs()`; + - the `'insert_kwargs_after'` key which its value must match one of ``*args``, + and any cmd-options will be appended after the matched arg. + + Examples:: - ``Examples``:: git.rev_list('master', max_count=10, header=True) + turns into:: + + git rev-list max-count 10 --header master + :return: Same as ``execute``""" # Handle optional arguments prior to calling transform_kwargs # otherwise these'll end up in args, which is bad. - _kwargs = dict() - for kwarg in execute_kwargs: - try: - _kwargs[kwarg] = kwargs.pop(kwarg) - except KeyError: - pass + exec_kwargs = {k: v for k, v in kwargs.items() if k in execute_kwargs} + opts_kwargs = {k: v for k, v in kwargs.items() if k not in execute_kwargs} - insert_after_this_arg = kwargs.pop('insert_kwargs_after', None) + insert_after_this_arg = opts_kwargs.pop('insert_kwargs_after', None) # Prepare the argument list - opt_args = self.transform_kwargs(**kwargs) + opt_args = self.transform_kwargs(**opts_kwargs) ext_args = self.__unpack_args([a for a in args if a is not None]) if insert_after_this_arg is None: @@ -798,55 +997,27 @@ def _call_process(self, method, *args, **kwargs): else: try: index = ext_args.index(insert_after_this_arg) - except ValueError: - raise ValueError("Couldn't find argument '%s' in args %s to insert kwargs after" - % (insert_after_this_arg, str(ext_args))) + except ValueError as err: + raise ValueError("Couldn't find argument '%s' in args %s to insert cmd options after" + % (insert_after_this_arg, str(ext_args))) from err # end handle error args = ext_args[:index + 1] + opt_args + ext_args[index + 1:] - # end handle kwargs + # end handle opts_kwargs - def make_call(): - call = [self.GIT_PYTHON_GIT_EXECUTABLE] + call = [self.GIT_PYTHON_GIT_EXECUTABLE] - # add the git options, the reset to empty - # to avoid side_effects - call.extend(self._git_options) - self._git_options = () + # add persistent git options + call.extend(self._persistent_git_options) - call.extend([dashify(method)]) - call.extend(args) - return call - # END utility to recreate call after changes + # add the git options, then reset to empty + # to avoid side_effects + call.extend(self._git_options) + self._git_options = () - if sys.platform == 'win32': - try: - try: - return self.execute(make_call(), **_kwargs) - except WindowsError: - # did we switch to git.cmd already, or was it changed from default ? permanently fail - if self.GIT_PYTHON_GIT_EXECUTABLE != self.git_exec_name: - raise - # END handle overridden variable - type(self).GIT_PYTHON_GIT_EXECUTABLE = self.git_exec_name_win + call.append(dashify(method)) + call.extend(args) - try: - return self.execute(make_call(), **_kwargs) - finally: - import warnings - msg = "WARNING: Automatically switched to use git.cmd as git executable" - msg += ", which reduces performance by ~70%." - msg += "It is recommended to put git.exe into the PATH or to " - msg += "set the %s " % self._git_exec_env_var - msg += "environment variable to the executable's location" - warnings.warn(msg) - # END print of warning - # END catch first failure - except WindowsError: - raise WindowsError("The system cannot find or execute the file at %r" % self.GIT_PYTHON_GIT_EXECUTABLE) - # END provide better error message - else: - return self.execute(make_call(), **_kwargs) - # END handle windows default installation + return self.execute(call, **exec_kwargs) def _parse_object_header(self, header_line): """ @@ -876,7 +1047,7 @@ def _prepare_ref(self, ref): if isinstance(ref, bytes): # Assume 40 bytes hexsha - bin-to-ascii for some reason returns bytes, not text refstr = ref.decode('ascii') - elif not isinstance(ref, string_types): + elif not isinstance(ref, str): refstr = str(ref) # could be ref-object if not refstr.endswith("\n"): @@ -935,6 +1106,10 @@ def clear_cache(self): Currently persistent commands will be interrupted. :return: self""" + for cmd in (self.cat_file_all, self.cat_file_header): + if cmd: + cmd.__del__() + self.cat_file_all = None self.cat_file_header = None return self diff --git a/git/compat.py b/git/compat.py index 1ea2119e1..a0aea1ac4 100644 --- a/git/compat.py +++ b/git/compat.py @@ -1,4 +1,4 @@ -#-*-coding:utf-8-*- +# -*- coding: utf-8 -*- # config.py # Copyright (C) 2008, 2009 Michael Trier (mtrier@gmail.com) and contributors # @@ -7,62 +7,75 @@ """utilities to help provide compatibility with python 3""" # flake8: noqa +import locale +import os import sys -from gitdb.utils.compat import ( - PY3, - xrange, - MAXSIZE, - izip, -) - from gitdb.utils.encoding import ( - string_types, - text_type, - force_bytes, - force_text + force_bytes, # @UnusedImport + force_text # @UnusedImport ) -defenc = sys.getdefaultencoding() -if PY3: - import io - FileType = io.IOBase - def byte_ord(b): - return b - def bchr(n): - return bytes([n]) - def mviter(d): - return d.values() - unicode = str -else: - FileType = file - # usually, this is just ascii, which might not enough for our encoding needs - # Unless it's set specifically, we override it to be utf-8 - if defenc == 'ascii': - defenc = 'utf-8' - byte_ord = ord - bchr = chr - unicode = unicode - def mviter(d): - return d.itervalues() - - -def with_metaclass(meta, *bases): +# typing -------------------------------------------------------------------- + +from typing import Any, AnyStr, Dict, Optional, Type +from git.types import TBD + +# --------------------------------------------------------------------------- + + +is_win = (os.name == 'nt') # type: bool +is_posix = (os.name == 'posix') +is_darwin = (os.name == 'darwin') +defenc = sys.getfilesystemencoding() + + +def safe_decode(s: Optional[AnyStr]) -> Optional[str]: + """Safely decodes a binary string to unicode""" + if isinstance(s, str): + return s + elif isinstance(s, bytes): + return s.decode(defenc, 'surrogateescape') + elif s is None: + return None + else: + raise TypeError('Expected bytes or text, but got %r' % (s,)) + + +def safe_encode(s: Optional[AnyStr]) -> Optional[bytes]: + """Safely encodes a binary string to unicode""" + if isinstance(s, str): + return s.encode(defenc) + elif isinstance(s, bytes): + return s + elif s is None: + return None + else: + raise TypeError('Expected bytes or text, but got %r' % (s,)) + + +def win_encode(s: Optional[AnyStr]) -> Optional[bytes]: + """Encode unicodes for process arguments on Windows.""" + if isinstance(s, str): + return s.encode(locale.getpreferredencoding(False)) + elif isinstance(s, bytes): + return s + elif s is not None: + raise TypeError('Expected bytes or text, but got %r' % (s,)) + return None + + + +def with_metaclass(meta: Type[Any], *bases: Any) -> 'metaclass': # type: ignore ## mypy cannot understand dynamic class creation """copied from https://github.com/Byron/bcore/blob/master/src/python/butility/future.py#L15""" - class metaclass(meta): + + class metaclass(meta): # type: ignore __call__ = type.__call__ - __init__ = type.__init__ + __init__ = type.__init__ # type: ignore - def __new__(cls, name, nbases, d): + def __new__(cls, name: str, nbases: Optional[int], d: Dict[str, Any]) -> TBD: if nbases is None: return type.__new__(cls, name, (), d) - # There may be clients who rely on this attribute to be set to a reasonable value, which is why - # we set the __metaclass__ attribute explicitly - if not PY3 and '___metaclass__' not in d: - d['__metaclass__'] = meta - # end return meta(name, bases, d) - # end - # end metaclass + return metaclass(meta.__name__ + 'Helper', None, {}) - # end handle py2 diff --git a/git/config.py b/git/config.py index b7ddf0d22..aadb0aac0 100644 --- a/git/config.py +++ b/git/config.py @@ -6,27 +6,30 @@ """Module containing module parser implementation able to properly read and write configuration files""" -import re -try: - import ConfigParser as cp -except ImportError: - # PY3 - import configparser as cp +import abc +from functools import wraps import inspect +from io import IOBase import logging -import abc import os +import re +import fnmatch +from collections import OrderedDict + +from typing_extensions import Literal -from git.odict import OrderedDict -from git.util import LockFile from git.compat import ( - string_types, - FileType, defenc, force_text, with_metaclass, - PY3 + is_win, ) +from git.util import LockFile + +import os.path as osp + +import configparser as cp + __all__ = ('GitConfigParser', 'SectionConstraint') @@ -34,11 +37,19 @@ log = logging.getLogger('git.config') log.addHandler(logging.NullHandler()) +# invariants +# represents the configuration level of a configuration file +CONFIG_LEVELS = ("system", "user", "global", "repository") + +# Section pattern to detect conditional includes. +# https://git-scm.com/docs/git-config#_conditional_includes +CONDITIONAL_INCLUDE_REGEXP = re.compile(r"(?<=includeIf )\"(gitdir|gitdir/i|onbranch):(.+)\"") + class MetaParserBuilder(abc.ABCMeta): """Utlity class wrapping base-class methods into decorators that assure read-only properties""" - def __new__(metacls, name, bases, clsdict): + def __new__(cls, name, bases, clsdict): """ Equip all base-class methods with a needs_values decorator, and all non-const methods with a set_dirty_and_flush_changes decorator in addition to that.""" @@ -60,18 +71,18 @@ def __new__(metacls, name, bases, clsdict): # END for each base # END if mutating methods configuration is set - new_type = super(MetaParserBuilder, metacls).__new__(metacls, name, bases, clsdict) + new_type = super(MetaParserBuilder, cls).__new__(cls, name, bases, clsdict) return new_type def needs_values(func): """Returns method assuring we read values (on demand) before we try to access them""" + @wraps(func) def assure_data_present(self, *args, **kwargs): self.read() return func(self, *args, **kwargs) # END wrapper method - assure_data_present.__name__ = func.__name__ return assure_data_present @@ -95,7 +106,10 @@ class SectionConstraint(object): """Constrains a ConfigParser to only option commands which are constrained to always use the section we have been initialized with. - It supports all ConfigParser methods that operate on an option""" + It supports all ConfigParser methods that operate on an option. + + :note: + If used as a context manager, will release the wrapped ConfigParser.""" __slots__ = ("_config", "_section_name") _valid_attrs_ = ("get_value", "set_value", "get", "set", "getint", "getfloat", "getboolean", "has_option", "remove_section", "remove_option", "options") @@ -129,6 +143,78 @@ def release(self): """Equivalent to GitConfigParser.release(), which is called on our underlying parser instance""" return self._config.release() + def __enter__(self): + self._config.__enter__() + return self + + def __exit__(self, exception_type, exception_value, traceback): + self._config.__exit__(exception_type, exception_value, traceback) + + +class _OMD(OrderedDict): + """Ordered multi-dict.""" + + def __setitem__(self, key, value): + super(_OMD, self).__setitem__(key, [value]) + + def add(self, key, value): + if key not in self: + super(_OMD, self).__setitem__(key, [value]) + return + + super(_OMD, self).__getitem__(key).append(value) + + def setall(self, key, values): + super(_OMD, self).__setitem__(key, values) + + def __getitem__(self, key): + return super(_OMD, self).__getitem__(key)[-1] + + def getlast(self, key): + return super(_OMD, self).__getitem__(key)[-1] + + def setlast(self, key, value): + if key not in self: + super(_OMD, self).__setitem__(key, [value]) + return + + prior = super(_OMD, self).__getitem__(key) + prior[-1] = value + + def get(self, key, default=None): + return super(_OMD, self).get(key, [default])[-1] + + def getall(self, key): + return super(_OMD, self).__getitem__(key) + + def items(self): + """List of (key, last value for key).""" + return [(k, self[k]) for k in self] + + def items_all(self): + """List of (key, list of values for key).""" + return [(k, self.getall(k)) for k in self] + + +def get_config_path(config_level: Literal['system', 'global', 'user', 'repository']) -> str: + + # we do not support an absolute path of the gitconfig on windows , + # use the global config instead + if is_win and config_level == "system": + config_level = "global" + + if config_level == "system": + return "/etc/gitconfig" + elif config_level == "user": + config_home = os.environ.get("XDG_CONFIG_HOME") or osp.join(os.environ.get("HOME", '~'), ".config") + return osp.normpath(osp.expanduser(osp.join(config_home, "git", "config"))) + elif config_level == "global": + return osp.normpath(osp.expanduser("~/.gitconfig")) + elif config_level == "repository": + raise ValueError("No repo to get repository configuration from. Use Repo._get_config_path") + + raise ValueError("Invalid configuration level: %r" % config_level) + class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, object)): @@ -145,30 +231,30 @@ class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, obje :note: The config is case-sensitive even when queried, hence section and option names - must match perfectly.""" + must match perfectly. + If used as a context manager, will release the locked file.""" #{ Configuration # The lock type determines the type of lock to use in new configuration readers. # They must be compatible to the LockFile interface. # A suitable alternative would be the BlockingLockFile t_lock = LockFile - re_comment = re.compile('^\s*[#;]') + re_comment = re.compile(r'^\s*[#;]') #} END configuration - OPTCRE = re.compile( - r'\s*(?P