diff --git a/.github/workflows/cygwin-test.yml b/.github/workflows/cygwin-test.yml index 61e6a3089..bde4ea659 100644 --- a/.github/workflows/cygwin-test.yml +++ b/.github/workflows/cygwin-test.yml @@ -40,6 +40,7 @@ jobs: - name: Special configuration for Cygwin git run: | git config --global --add safe.directory "$(pwd)" + git config --global --add safe.directory "$(pwd)/.git" git config --global core.autocrlf false - name: Prepare this repo for tests diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 4c918a92d..747db62f0 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,15 +13,11 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "macos-13", "macos-14", "windows-latest"] + os: ["ubuntu-22.04", "macos-latest", "windows-latest"] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] exclude: - - os: "macos-14" + - os: "macos-latest" python-version: "3.7" - - os: "macos-14" - python-version: "3.8" - - os: "macos-14" - python-version: "3.9" include: - experimental: false @@ -44,9 +40,10 @@ jobs: - name: Set up WSL (Windows) if: startsWith(matrix.os, 'windows') - uses: Vampire/setup-wsl@v3.0.0 + uses: Vampire/setup-wsl@v3.1.1 with: - distribution: Debian + distribution: Alpine + additional-packages: bash - name: Prepare this repo for tests run: | @@ -102,6 +99,7 @@ jobs: continue-on-error: false - name: Documentation + if: matrix.python-version != '3.7' run: | pip install ".[doc]" make -C doc html diff --git a/.gitignore b/.gitignore index 7765293d8..d85569405 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,6 @@ output.txt # Finder metadata .DS_Store + +# Files created by OSS-Fuzz when running locally +fuzz_*.pkg.spec diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 585b4f04d..424cc5f37 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,38 @@ repos: +- repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + additional_dependencies: [tomli] + exclude: ^test/fixtures/ + - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.2 + rev: v0.6.0 hooks: - - id: ruff-format - exclude: ^git/ext/ - id: ruff args: ["--fix"] exclude: ^git/ext/ + - id: ruff-format + exclude: ^git/ext/ - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.6 + rev: v0.10.0.1 hooks: - id: shellcheck args: [--color] exclude: ^test/fixtures/polyglot$|^git/ext/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: + - id: end-of-file-fixer + exclude: ^test/fixtures/|COPYING|LICENSE + - id: check-symlinks - id: check-toml - id: check-yaml - id: check-merge-conflict + +- repo: https://github.com/abravalheri/validate-pyproject + rev: v0.19 + hooks: + - id: validate-pyproject diff --git a/AUTHORS b/AUTHORS index 9311b3962..45b14c961 100644 --- a/AUTHORS +++ b/AUTHORS @@ -54,5 +54,6 @@ Contributors are: -Wenhan Zhu -Eliah Kagan -Ethan Lin +-Jonas Scharpf Portions derived from other open source works and are clearly marked. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e108f1b80..8536d7f73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,3 +8,8 @@ The following is a short step-by-step rundown of what one typically would do to - Try to avoid massive commits and prefer to take small steps, with one commit for each. - Feel free to add yourself to AUTHORS file. - Create a pull request. + +## Fuzzing Test Specific Documentation + +For details related to contributing to the fuzzing test suite and OSS-Fuzz integration, please +refer to the dedicated [fuzzing README](./fuzzing/README.md). diff --git a/README.md b/README.md index 9bedaaae7..59c6f995b 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ In the less common case that you do not want to install test dependencies, `pip #### With editable *dependencies* (not preferred, and rarely needed) -In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediatley reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython. +In rare cases, you may want to work on GitPython and one or both of its [gitdb](https://github.com/gitpython-developers/gitdb) and [smmap](https://github.com/gitpython-developers/smmap) dependencies at the same time, with changes in your local working copy of gitdb or smmap immediately reflected in the behavior of your local working copy of GitPython. This can be done by making editable installations of those dependencies in the same virtual environment where you install GitPython. If you want to do that *and* you want the versions in GitPython's git submodules to be used, then pass `-e git/ext/gitdb` and/or `-e git/ext/gitdb/gitdb/ext/smmap` to `pip install`. This can be done in any order, and in separate `pip install` commands or the same one, so long as `-e` appears before *each* path. For example, you can install GitPython, gitdb, and smmap editably in the currently active virtual environment this way: @@ -240,5 +240,8 @@ Please have a look at the [contributions file][contributing]. [3-Clause BSD License](https://opensource.org/license/bsd-3-clause/), also known as the New BSD License. See the [LICENSE file][license]. +One file exclusively used for fuzz testing is subject to [a separate license, detailed here](./fuzzing/README.md#license). +This file is not included in the wheel or sdist packages published by the maintainers of GitPython. + [contributing]: https://github.com/gitpython-developers/GitPython/blob/main/CONTRIBUTING.md [license]: https://github.com/gitpython-developers/GitPython/blob/main/LICENSE diff --git a/VERSION b/VERSION index d1bf6638d..e6af1c454 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.43 +3.1.44 diff --git a/build-release.sh b/build-release.sh index 49c13b93a..1a8dce2c2 100755 --- a/build-release.sh +++ b/build-release.sh @@ -1,5 +1,8 @@ #!/bin/bash # +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +# # This script builds a release. If run in a venv, it auto-installs its tools. # You may want to run "make release" instead of running this script directly. diff --git a/check-version.sh b/check-version.sh index dac386e46..579cf789f 100755 --- a/check-version.sh +++ b/check-version.sh @@ -1,5 +1,8 @@ #!/bin/bash # +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ +# # This script checks if we are in a consistent state to build a new release. # See the release instructions in README.md for the steps to make this pass. # You may want to run "make release" instead of running this script directly. diff --git a/doc/requirements.txt b/doc/requirements.txt index 7769af5ae..81140d898 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,8 +1,3 @@ -sphinx == 4.3.2 +sphinx >= 7.1.2, < 7.2 sphinx_rtd_theme -sphinxcontrib-applehelp >= 1.0.2, <= 1.0.4 -sphinxcontrib-devhelp == 1.0.2 -sphinxcontrib-htmlhelp >= 2.0.0, <= 2.0.1 -sphinxcontrib-qthelp == 1.0.3 -sphinxcontrib-serializinghtml == 1.1.5 sphinx-autodoc-typehints diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 0bc757134..00a3c660e 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,12 @@ Changelog ========= +3.1.44 +====== + +See the following for all changes. +https://github.com/gitpython-developers/GitPython/releases/tag/3.1.44 + 3.1.43 ====== @@ -20,7 +26,7 @@ https://github.com/gitpython-developers/GitPython/releases/tag/3.1.42 3.1.41 ====== -This release is relevant for security as it fixes a possible arbitary +This release is relevant for security as it fixes a possible arbitrary code execution on Windows. See this PR for details: https://github.com/gitpython-developers/GitPython/pull/1792 diff --git a/doc/source/index.rst b/doc/source/index.rst index 72db8ee5a..ca5229ac3 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -21,4 +21,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/doc/source/intro.rst b/doc/source/intro.rst index 4f22a0942..d053bd117 100644 --- a/doc/source/intro.rst +++ b/doc/source/intro.rst @@ -122,4 +122,3 @@ License Information =================== GitPython is licensed under the New BSD License. See the LICENSE file for more information. - diff --git a/doc/source/roadmap.rst b/doc/source/roadmap.rst index a573df33a..34c953626 100644 --- a/doc/source/roadmap.rst +++ b/doc/source/roadmap.rst @@ -6,4 +6,3 @@ The full list of milestones including associated tasks can be found on GitHub: https://github.com/gitpython-developers/GitPython/issues Select the respective milestone to filter the list of issues accordingly. - diff --git a/fuzzing/LICENSE-APACHE b/fuzzing/LICENSE-APACHE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/fuzzing/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/fuzzing/LICENSE-BSD b/fuzzing/LICENSE-BSD new file mode 120000 index 000000000..ea5b60640 --- /dev/null +++ b/fuzzing/LICENSE-BSD @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/fuzzing/README.md b/fuzzing/README.md new file mode 100644 index 000000000..286f529eb --- /dev/null +++ b/fuzzing/README.md @@ -0,0 +1,226 @@ +# Fuzzing GitPython + +[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/gitpython.svg)][oss-fuzz-issue-tracker] + +This directory contains files related to GitPython's suite of fuzz tests that are executed daily on automated +infrastructure provided by [OSS-Fuzz][oss-fuzz-repo]. This document aims to provide necessary information for working +with fuzzing in GitPython. + +The latest details regarding OSS-Fuzz test status, including build logs and coverage reports, is available +on [the Open Source Fuzzing Introspection website](https://introspector.oss-fuzz.com/project-profile?project=gitpython). + +## How to Contribute + +There are many ways to contribute to GitPython's fuzzing efforts! Contributions are welcomed through issues, +discussions, or pull requests on this repository. + +Areas that are particularly appreciated include: + +- **Tackling the existing backlog of open issues**. While fuzzing is an effective way to identify bugs, that information + isn't useful unless they are fixed. If you are not sure where to start, the issues tab is a great place to get ideas! +- **Improvements to this (or other) documentation** make it easier for new contributors to get involved, so even small + improvements can have a large impact over time. If you see something that could be made easier by a documentation + update of any size, please consider suggesting it! + +For everything else, such as expanding test coverage, optimizing test performance, or enhancing error detection +capabilities, jump into the "Getting Started" section below. + +## Getting Started with Fuzzing GitPython + +> [!TIP] +> **New to fuzzing or unfamiliar with OSS-Fuzz?** +> +> These resources are an excellent place to start: +> +> - [OSS-Fuzz documentation][oss-fuzz-docs] - Continuous fuzzing service for open source software. +> - [Google/fuzzing][google-fuzzing-repo] - Tutorials, examples, discussions, research proposals, and other resources + related to fuzzing. +> - [CNCF Fuzzing Handbook](https://github.com/cncf/tag-security/blob/main/security-fuzzing-handbook/handbook-fuzzing.pdf) - + A comprehensive guide for fuzzing open source software. +> - [Efficient Fuzzing Guide by The Chromium Project](https://chromium.googlesource.com/chromium/src/+/main/testing/libfuzzer/efficient_fuzzing.md) - + Explores strategies to enhance the effectiveness of your fuzz tests, recommended for those looking to optimize their + testing efforts. + +### Setting Up Your Local Environment + +Before contributing to fuzzing efforts, ensure Python and Docker are installed on your machine. Docker is required for +running fuzzers in containers provided by OSS-Fuzz and for safely executing test files directly. [Install Docker](https://docs.docker.com/get-docker/) following the official guide if you do not already have it. + +### Understanding Existing Fuzz Targets + +Review the `fuzz-targets/` directory to familiarize yourself with how existing tests are implemented. See +the [Files & Directories Overview](#files--directories-overview) for more details on the directory structure. + +### Contributing to Fuzz Tests + +Start by reviewing the [Atheris documentation][atheris-repo] and the section +on [Running Fuzzers Locally](#running-fuzzers-locally) to begin writing or improving fuzz tests. + +## Files & Directories Overview + +The `fuzzing/` directory is organized into three key areas: + +### Fuzz Targets (`fuzz-targets/`) + +Contains Python files for each fuzz test. + +**Things to Know**: + +- Each fuzz test targets a specific part of GitPython's functionality. +- Test files adhere to the naming convention: `fuzz_.py`, where `` indicates the + functionality targeted by the test. +- Any functionality that involves performing operations on input data is a possible candidate for fuzz testing, but + features that involve processing untrusted user input or parsing operations are typically going to be the most + interesting. +- The goal of these tests is to identify previously unknown or unexpected error cases caused by a given input. For that + reason, fuzz tests should gracefully handle anticipated exception cases with a `try`/`except` block to avoid false + positives that halt the fuzzing engine. + +### OSS-Fuzz Scripts (`oss-fuzz-scripts/`) + +Includes scripts for building and integrating fuzz targets with OSS-Fuzz: + +- **`container-environment-bootstrap.sh`** - Sets up the execution environment. It is responsible for fetching default + dictionary entries and ensuring all required build dependencies are installed and up-to-date. +- **`build.sh`** - Executed within the Docker container, this script builds fuzz targets with necessary instrumentation + and prepares seed corpora and dictionaries for use. + +**Where to learn more:** + +- [OSS-Fuzz documentation on the build.sh](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#buildsh) +- [See GitPython's build.sh and Dockerfile in the OSS-Fuzz repository](https://github.com/google/oss-fuzz/tree/master/projects/gitpython) + +### Local Development Helpers (`local-dev-helpers/`) + +Contains tools to make local development tasks easier. +See [the "Running Fuzzers Locally" section below](#running-fuzzers-locally) for further documentation and use cases related to files found here. + +## Running Fuzzers Locally + +> [!WARNING] +> **Some fuzz targets in this repository write to the filesystem** during execution. +> For that reason, it is strongly recommended to **always use Docker when executing fuzz targets**, even when it may be +> possible to do so without it. +> +> Although [I/O operations such as writing to disk are not considered best practice](https://github.com/google/fuzzing/blob/master/docs/good-fuzz-target.md#io), the current implementation of at least one test requires it. +> See [the "Setting Up Your Local Environment" section above](#setting-up-your-local-environment) if you do not already have Docker installed on your machine. +> +> PRs that replace disk I/O with in-memory alternatives are very much welcomed! + +### Direct Execution of Fuzz Targets + +Directly executing fuzz targets allows for quick iteration and testing of changes which can be helpful during early +development of new fuzz targets or for validating changes made to an existing test. +The [Dockerfile](./local-dev-helpers/Dockerfile) located in the `local-dev-helpers/` subdirectory provides a lightweight +container environment preconfigured with [Atheris][atheris-repo] that makes it easy to execute a fuzz target directly. + +**From the root directory of your GitPython repository clone**: + +1. Build the local development helper image: + +```shell +docker build -f fuzzing/local-dev-helpers/Dockerfile -t gitpython-fuzzdev . +``` + +2. Then execute a fuzz target inside the image, for example: + +```shell + docker run -it -v "$PWD":/src gitpython-fuzzdev python fuzzing/fuzz-targets/fuzz_config.py -atheris_runs=10000 +``` + +The above command executes [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) and exits after `10000` runs, or earlier if +the fuzzer finds an error. + +Docker CLI's `-v` flag specifies a volume mount in Docker that maps the directory in which the command is run (which +should be the root directory of your local GitPython clone) to a directory inside the container, so any modifications +made between invocations will be reflected immediately without the need to rebuild the image each time. + +### Running OSS-Fuzz Locally + +This approach uses Docker images provided by OSS-Fuzz for building and running fuzz tests locally. It offers +comprehensive features but requires a local clone of the OSS-Fuzz repository and sufficient disk space for Docker +containers. + +#### Build the Execution Environment + +Clone the OSS-Fuzz repository and prepare the Docker environment: + +```shell +git clone --depth 1 https://github.com/google/oss-fuzz.git oss-fuzz +cd oss-fuzz +python infra/helper.py build_image gitpython +python infra/helper.py build_fuzzers --sanitizer address gitpython +``` + +> [!TIP] +> The `build_fuzzers` command above accepts a local file path pointing to your GitPython repository clone as the last +> argument. +> This makes it easy to build fuzz targets you are developing locally in this repository without changing anything in +> the OSS-Fuzz repo! +> For example, if you have cloned this repository (or a fork of it) into: `~/code/GitPython` +> Then running this command would build new or modified fuzz targets using the `~/code/GitPython/fuzzing/fuzz-targets` +> directory: +> ```shell +> python infra/helper.py build_fuzzers --sanitizer address gitpython ~/code/GitPython +> ``` + +Verify the build of your fuzzers with the optional `check_build` command: + +```shell +python infra/helper.py check_build gitpython +``` + +#### Run a Fuzz Target + +Setting an environment variable for the fuzz target argument of the execution command makes it easier to quickly select +a different target between runs: + +```shell +# specify the fuzz target without the .py extension: +export FUZZ_TARGET=fuzz_config +``` + +Execute the desired fuzz target: + +```shell +python infra/helper.py run_fuzzer gitpython $FUZZ_TARGET -- -max_total_time=60 -print_final_stats=1 +``` + +> [!TIP] +> In the example above, the "`-- -max_total_time=60 -print_final_stats=1`" portion of the command is optional but quite +> useful. +> +> Every argument provided after "`--`" in the above command is passed to the fuzzing engine directly. In this case: +> - `-max_total_time=60` tells the LibFuzzer to stop execution after 60 seconds have elapsed. +> - `-print_final_stats=1` tells the LibFuzzer to print a summary of useful metrics about the target run upon + completion. +> +> But almost any [LibFuzzer option listed in the documentation](https://llvm.org/docs/LibFuzzer.html#options) should +> work as well. + +#### Next Steps + +For detailed instructions on advanced features like reproducing OSS-Fuzz issues or using the Fuzz Introspector, refer +to [the official OSS-Fuzz documentation][oss-fuzz-docs]. + +## LICENSE + +All files located within the `fuzzing/` directory are subject to [the same license](../LICENSE) +as [the other files in this repository](../README.md#license) with one exception: + +[`fuzz_config.py`](./fuzz-targets/fuzz_config.py) was migrated to this repository from the OSS-Fuzz project's repository +where it was originally created. As such, [`fuzz_config.py`](./fuzz-targets/fuzz_config.py) retains its original license +and copyright notice (Apache License, Version 2.0 and Copyright 2023 Google LLC respectively) as in a header +comment, followed by a notice stating that it has have been modified contributors to GitPython. +[LICENSE-APACHE](./LICENSE-APACHE) contains the original license used by the OSS-Fuzz project repository at the time the +file was migrated. + +[oss-fuzz-repo]: https://github.com/google/oss-fuzz + +[oss-fuzz-docs]: https://google.github.io/oss-fuzz + +[oss-fuzz-issue-tracker]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:gitpython + +[google-fuzzing-repo]: https://github.com/google/fuzzing + +[atheris-repo]: https://github.com/google/atheris diff --git a/fuzzing/fuzz-targets/fuzz_blob.py b/fuzzing/fuzz-targets/fuzz_blob.py new file mode 100644 index 000000000..ce888e85f --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_blob.py @@ -0,0 +1,40 @@ +import atheris +import sys +import os +import tempfile + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + import git + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = git.Repo.init(path=temp_dir) + binsha = fdp.ConsumeBytes(20) + mode = fdp.ConsumeInt(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())) + path = fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes()) + + try: + blob = git.Blob(repo, binsha, mode, path) + except AssertionError as e: + if "Require 20 byte binary sha, got" in str(e): + return -1 + else: + raise e + + _ = blob.mime_type + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_config.py b/fuzzing/fuzz-targets/fuzz_config.py new file mode 100644 index 000000000..4eddc32ff --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_config.py @@ -0,0 +1,57 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################### +# Note: This file has been modified by contributors to GitPython. +# The original state of this file may be referenced here: +# https://github.com/google/oss-fuzz/commit/f26f254558fc48f3c9bc130b10507386b94522da +############################################################################### +import atheris +import sys +import io +import os +from configparser import MissingSectionHeaderError, ParsingError + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + import git + + +def TestOneInput(data): + sio = io.BytesIO(data) + sio.name = "/tmp/fuzzconfig.config" + git_config = git.GitConfigParser(sio) + try: + git_config.read() + except (MissingSectionHeaderError, ParsingError, UnicodeDecodeError): + return -1 # Reject inputs raising expected exceptions + except ValueError as e: + if "embedded null byte" in str(e): + # The `os.path.expanduser` function, which does not accept strings + # containing null bytes might raise this. + return -1 + else: + raise e # Raise unanticipated exceptions as they might be bugs + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_diff.py b/fuzzing/fuzz-targets/fuzz_diff.py new file mode 100644 index 000000000..d4bd68b57 --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_diff.py @@ -0,0 +1,86 @@ +import sys +import os +import io +import tempfile +from binascii import Error as BinasciiError + +import atheris + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + from git import Repo, Diff + + +class BytesProcessAdapter: + """Allows bytes to be used as process objects returned by subprocess.Popen.""" + + @atheris.instrument_func + def __init__(self, input_string): + self.stdout = io.BytesIO(input_string) + self.stderr = io.BytesIO() + + @atheris.instrument_func + def wait(self): + return 0 + + poll = wait + + +@atheris.instrument_func +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = Repo.init(path=temp_dir) + try: + diff = Diff( + repo, + a_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + b_rawpath=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + a_blob_id=fdp.ConsumeBytes(20), + b_blob_id=fdp.ConsumeBytes(20), + a_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + b_mode=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + new_file=fdp.ConsumeBool(), + deleted_file=fdp.ConsumeBool(), + copied_file=fdp.ConsumeBool(), + raw_rename_from=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + raw_rename_to=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + diff=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), + change_type=fdp.PickValueInList(["A", "D", "C", "M", "R", "T", "U"]), + score=fdp.ConsumeIntInRange(0, fdp.remaining_bytes()), + ) + except BinasciiError: + return -1 + except AssertionError as e: + if "Require 20 byte binary sha, got" in str(e): + return -1 + else: + raise e + + _ = diff.__str__() + _ = diff.a_path + _ = diff.b_path + _ = diff.rename_from + _ = diff.rename_to + _ = diff.renamed_file + + diff_index = diff._index_from_patch_format( + repo, proc=BytesProcessAdapter(fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes()))) + ) + + diff._handle_diff_line( + lines_bytes=fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, fdp.remaining_bytes())), repo=repo, index=diff_index + ) + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_repo.py b/fuzzing/fuzz-targets/fuzz_repo.py new file mode 100644 index 000000000..7bd82c120 --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_repo.py @@ -0,0 +1,47 @@ +import atheris +import io +import sys +import os +import tempfile + +if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + path_to_bundled_git_binary = os.path.abspath(os.path.join(os.path.dirname(__file__), "git")) + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = path_to_bundled_git_binary + +with atheris.instrument_imports(): + import git + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as temp_dir: + repo = git.Repo.init(path=temp_dir) + + # Generate a minimal set of files based on fuzz data to minimize I/O operations. + file_paths = [os.path.join(temp_dir, f"File{i}") for i in range(min(3, fdp.ConsumeIntInRange(1, 3)))] + for file_path in file_paths: + with open(file_path, "wb") as f: + # The chosen upperbound for count of bytes we consume by writing to these + # files is somewhat arbitrary and may be worth experimenting with if the + # fuzzer coverage plateaus. + f.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) + + repo.index.add(file_paths) + repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 80))) + + fuzz_tree = git.Tree(repo, git.Tree.NULL_BIN_SHA, 0, "") + + try: + fuzz_tree._deserialize(io.BytesIO(data)) + except IndexError: + return -1 + + +def main(): + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/fuzz_submodule.py b/fuzzing/fuzz-targets/fuzz_submodule.py new file mode 100644 index 000000000..d22b0aa5b --- /dev/null +++ b/fuzzing/fuzz-targets/fuzz_submodule.py @@ -0,0 +1,100 @@ +import atheris +import sys +import os +import tempfile +from configparser import ParsingError +from utils import ( + setup_git_environment, + handle_exception, + get_max_filename_length, +) + +# Setup the git environment +setup_git_environment() +from git import Repo, GitCommandError, InvalidGitRepositoryError + + +def TestOneInput(data): + fdp = atheris.FuzzedDataProvider(data) + + with tempfile.TemporaryDirectory() as repo_temp_dir: + repo = Repo.init(path=repo_temp_dir) + repo.index.commit("Initial commit") + + try: + with tempfile.TemporaryDirectory() as submodule_temp_dir: + sub_repo = Repo.init(submodule_temp_dir, bare=fdp.ConsumeBool()) + sub_repo.index.commit(fdp.ConsumeUnicodeNoSurrogates(fdp.ConsumeIntInRange(1, 512))) + + submodule_name = fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(repo.working_tree_dir))) + ) + submodule_path = os.path.join(repo.working_tree_dir, submodule_name) + + submodule = repo.create_submodule(submodule_name, submodule_path, url=sub_repo.git_dir) + repo.index.commit("Added submodule") + + with submodule.config_writer() as writer: + key_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) + value_length = fdp.ConsumeIntInRange(1, max(1, fdp.remaining_bytes())) + + writer.set_value( + fdp.ConsumeUnicodeNoSurrogates(key_length), fdp.ConsumeUnicodeNoSurrogates(value_length) + ) + writer.release() + + submodule.update(init=fdp.ConsumeBool(), dry_run=fdp.ConsumeBool(), force=fdp.ConsumeBool()) + submodule_repo = submodule.module() + + new_file_name = fdp.ConsumeUnicodeNoSurrogates( + fdp.ConsumeIntInRange(1, max(1, get_max_filename_length(submodule_repo.working_tree_dir))) + ) + new_file_path = os.path.join(submodule_repo.working_tree_dir, new_file_name) + with open(new_file_path, "wb") as new_file: + new_file.write(fdp.ConsumeBytes(fdp.ConsumeIntInRange(1, 512))) + submodule_repo.index.add([new_file_path]) + submodule_repo.index.commit("Added new file to submodule") + + repo.submodule_update(recursive=fdp.ConsumeBool()) + submodule_repo.head.reset(commit="HEAD~1", working_tree=fdp.ConsumeBool(), head=fdp.ConsumeBool()) + # Use fdp.PickValueInList to ensure at least one of 'module' or 'configuration' is True + module_option_value, configuration_option_value = fdp.PickValueInList( + [(True, False), (False, True), (True, True)] + ) + submodule.remove( + module=module_option_value, + configuration=configuration_option_value, + dry_run=fdp.ConsumeBool(), + force=fdp.ConsumeBool(), + ) + repo.index.commit(f"Removed submodule {submodule_name}") + + except ( + ParsingError, + GitCommandError, + InvalidGitRepositoryError, + FileNotFoundError, + FileExistsError, + IsADirectoryError, + NotADirectoryError, + BrokenPipeError, + PermissionError, + ): + return -1 + except Exception as e: + if isinstance(e, ValueError) and "embedded null byte" in str(e): + return -1 + elif isinstance(e, OSError) and "File name too long" in str(e): + return -1 + else: + return handle_exception(e) + + +def main(): + atheris.instrument_all() + atheris.Setup(sys.argv, TestOneInput) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzzing/fuzz-targets/utils.py b/fuzzing/fuzz-targets/utils.py new file mode 100644 index 000000000..97e6eab98 --- /dev/null +++ b/fuzzing/fuzz-targets/utils.py @@ -0,0 +1,122 @@ +import atheris # pragma: no cover +import os # pragma: no cover +import re # pragma: no cover +import traceback # pragma: no cover +import sys # pragma: no cover +from typing import Set, Tuple, List # pragma: no cover + + +@atheris.instrument_func +def is_expected_exception_message(exception: Exception, error_message_list: List[str]) -> bool: # pragma: no cover + """ + Checks if the message of a given exception matches any of the expected error messages, case-insensitively. + + Args: + exception (Exception): The exception object raised during execution. + error_message_list (List[str]): A list of error message substrings to check against the exception's message. + + Returns: + bool: True if the exception's message contains any of the substrings from the error_message_list, + case-insensitively, otherwise False. + """ + exception_message = str(exception).lower() + for error in error_message_list: + if error.lower() in exception_message: + return True + return False + + +@atheris.instrument_func +def get_max_filename_length(path: str) -> int: # pragma: no cover + """ + Get the maximum filename length for the filesystem containing the given path. + + Args: + path (str): The path to check the filesystem for. + + Returns: + int: The maximum filename length. + """ + return os.pathconf(path, "PC_NAME_MAX") + + +@atheris.instrument_func +def read_lines_from_file(file_path: str) -> list: + """Read lines from a file and return them as a list.""" + try: + with open(file_path, "r") as f: + return [line.strip() for line in f if line.strip()] + except FileNotFoundError: + print(f"File not found: {file_path}") + return [] + except IOError as e: + print(f"Error reading file {file_path}: {e}") + return [] + + +@atheris.instrument_func +def load_exception_list(file_path: str = "explicit-exceptions-list.txt") -> Set[Tuple[str, str]]: + """Load and parse the exception list from a default or specified file.""" + try: + bundle_dir = os.path.dirname(os.path.abspath(__file__)) + full_path = os.path.join(bundle_dir, file_path) + lines = read_lines_from_file(full_path) + exception_list: Set[Tuple[str, str]] = set() + for line in lines: + match = re.match(r"(.+):(\d+):", line) + if match: + file_path: str = match.group(1).strip() + line_number: str = str(match.group(2).strip()) + exception_list.add((file_path, line_number)) + return exception_list + except Exception as e: + print(f"Error loading exception list: {e}") + return set() + + +@atheris.instrument_func +def match_exception_with_traceback(exception_list: Set[Tuple[str, str]], exc_traceback) -> bool: + """Match exception traceback with the entries in the exception list.""" + for filename, lineno, _, _ in traceback.extract_tb(exc_traceback): + for file_pattern, line_pattern in exception_list: + # Ensure filename and line_number are strings for regex matching + if re.fullmatch(file_pattern, filename) and re.fullmatch(line_pattern, str(lineno)): + return True + return False + + +@atheris.instrument_func +def check_exception_against_list(exc_traceback, exception_file: str = "explicit-exceptions-list.txt") -> bool: + """Check if the exception traceback matches any entry in the exception list.""" + exception_list = load_exception_list(exception_file) + return match_exception_with_traceback(exception_list, exc_traceback) + + +@atheris.instrument_func +def handle_exception(e: Exception) -> int: + """Encapsulate exception handling logic for reusability.""" + exc_traceback = e.__traceback__ + if check_exception_against_list(exc_traceback): + return -1 + else: + raise e + + +@atheris.instrument_func +def setup_git_environment() -> None: + """Set up the environment variables for Git.""" + bundle_dir = os.path.dirname(os.path.abspath(__file__)) + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): # pragma: no cover + bundled_git_binary_path = os.path.join(bundle_dir, "git") + os.environ["GIT_PYTHON_GIT_EXECUTABLE"] = bundled_git_binary_path + + if not sys.warnoptions: # pragma: no cover + # The warnings filter below can be overridden by passing the -W option + # to the Python interpreter command line or setting the `PYTHONWARNINGS` environment variable. + import warnings + import logging + + # Fuzzing data causes some modules to generate a large number of warnings + # which are not usually interesting and make the test output hard to read, so we ignore them. + warnings.simplefilter("ignore") + logging.getLogger().setLevel(logging.ERROR) diff --git a/fuzzing/local-dev-helpers/Dockerfile b/fuzzing/local-dev-helpers/Dockerfile new file mode 100644 index 000000000..426de05dd --- /dev/null +++ b/fuzzing/local-dev-helpers/Dockerfile @@ -0,0 +1,22 @@ +# syntax=docker/dockerfile:1 + +# Use the same Python version as OSS-Fuzz to accidental incompatibilities in test code +FROM python:3.8-bookworm + +LABEL project="GitPython Fuzzing Local Dev Helper" + +WORKDIR /src + +COPY . . + +# Update package managers, install necessary packages, and cleanup unnecessary files in a single RUN to keep the image smaller. +RUN apt-get update && \ + apt-get install -y git clang && \ + python -m pip install --upgrade pip && \ + python -m pip install atheris && \ + python -m pip install -e . && \ + apt-get clean && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + +CMD ["bash"] diff --git a/fuzzing/oss-fuzz-scripts/build.sh b/fuzzing/oss-fuzz-scripts/build.sh new file mode 100644 index 000000000..c156e872d --- /dev/null +++ b/fuzzing/oss-fuzz-scripts/build.sh @@ -0,0 +1,19 @@ +# shellcheck shell=bash +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +set -euo pipefail + +python3 -m pip install . + +find "$SRC" -maxdepth 1 \ + \( -name '*_seed_corpus.zip' -o -name '*.options' -o -name '*.dict' \) \ + -exec printf '[%s] Copying: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" {} \; \ + -exec chmod a-x {} \; \ + -exec cp {} "$OUT" \; + +# Build fuzzers in $OUT. +find "$SRC/gitpython/fuzzing" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do + compile_python_fuzzer "$fuzz_harness" --add-binary="$(command -v git):." --add-data="$SRC/explicit-exceptions-list.txt:." +done diff --git a/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh new file mode 100755 index 000000000..924a3cbf3 --- /dev/null +++ b/fuzzing/oss-fuzz-scripts/container-environment-bootstrap.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ + +set -euo pipefail + +################# +# Prerequisites # +################# + +for cmd in python3 git wget zip; do + command -v "$cmd" >/dev/null 2>&1 || { + printf '[%s] Required command %s not found, exiting.\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$cmd" >&2 + exit 1 + } +done + +############# +# Functions # +############# + +download_and_concatenate_common_dictionaries() { + # Assign the first argument as the target file where all contents will be concatenated + local target_file="$1" + + # Shift the arguments so the first argument (target_file path) is removed + # and only URLs are left for the loop below. + shift + + for url in "$@"; do + wget -qO- "$url" >>"$target_file" + # Ensure there's a newline between each file's content + echo >>"$target_file" + done +} + +create_seed_corpora_zips() { + local seed_corpora_dir="$1" + local output_zip + for dir in "$seed_corpora_dir"/*; do + if [ -d "$dir" ] && [ -n "$dir" ]; then + output_zip="$SRC/$(basename "$dir")_seed_corpus.zip" + printf '[%s] Zipping the contents of %s into %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dir" "$output_zip" + zip -jur "$output_zip" "$dir"/* + fi + done +} + +prepare_dictionaries_for_fuzz_targets() { + local dictionaries_dir="$1" + local fuzz_targets_dir="$2" + local common_base_dictionary_filename="$WORK/__base.dict" + + printf '[%s] Copying .dict files from %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$dictionaries_dir" "$SRC/" + cp -v "$dictionaries_dir"/*.dict "$SRC/" + + download_and_concatenate_common_dictionaries "$common_base_dictionary_filename" \ + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/utf8.dict" \ + "https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/url.dict" + + find "$fuzz_targets_dir" -name 'fuzz_*.py' -print0 | while IFS= read -r -d '' fuzz_harness; do + if [[ -r "$common_base_dictionary_filename" ]]; then + # Strip the `.py` extension from the filename and replace it with `.dict`. + fuzz_harness_dictionary_filename="$(basename "$fuzz_harness" .py).dict" + local output_file="$SRC/$fuzz_harness_dictionary_filename" + + printf '[%s] Appending %s to %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$common_base_dictionary_filename" "$output_file" + if [[ -s "$output_file" ]]; then + # If a dictionary file for this fuzzer already exists and is not empty, + # we append a new line to the end of it before appending any new entries. + # + # LibFuzzer will happily ignore multiple empty lines in a dictionary but fail with an error + # if any single line has incorrect syntax (e.g., if we accidentally add two entries to the same line.) + # See docs for valid syntax: https://llvm.org/docs/LibFuzzer.html#id32 + echo >>"$output_file" + fi + cat "$common_base_dictionary_filename" >>"$output_file" + fi + done +} + +######################## +# Main execution logic # +######################## +# Seed corpora and dictionaries are hosted in a separate repository to avoid additional bloat in this repo. +# We clone into the $WORK directory because OSS-Fuzz cleans it up after building the image, keeping the image small. +git clone --depth 1 https://github.com/gitpython-developers/qa-assets.git "$WORK/qa-assets" + +create_seed_corpora_zips "$WORK/qa-assets/gitpython/corpora" + +prepare_dictionaries_for_fuzz_targets "$WORK/qa-assets/gitpython/dictionaries" "$SRC/gitpython/fuzzing" + +pushd "$SRC/gitpython/" +# Search for 'raise' and 'assert' statements in Python files within GitPython's source code and submodules, saving the +# matched file path, line number, and line content to a file named 'explicit-exceptions-list.txt'. +# This file can then be used by fuzz harnesses to check exception tracebacks and filter out explicitly raised or otherwise +# anticipated exceptions to reduce false positive test failures. + +git grep -n --recurse-submodules -e '\braise\b' -e '\bassert\b' -- '*.py' -- ':!setup.py' -- ':!test/**' -- ':!fuzzing/**' > "$SRC/explicit-exceptions-list.txt" + +popd + + +# The OSS-Fuzz base image has outdated dependencies by default so we upgrade them below. +python3 -m pip install --upgrade pip +# Upgrade to the latest versions known to work at the time the below changes were introduced: +python3 -m pip install 'setuptools~=69.0' 'pyinstaller~=6.0' diff --git a/git/cmd.py b/git/cmd.py index 90fc39cd6..2048a43fa 100644 --- a/git/cmd.py +++ b/git/cmd.py @@ -1269,6 +1269,7 @@ def execute( stdout=stdout_sink, shell=shell, universal_newlines=universal_newlines, + encoding=defenc if universal_newlines else None, **subprocess_kwargs, ) except cmd_not_found_exception as err: diff --git a/git/config.py b/git/config.py index 3ce9b123f..de3508360 100644 --- a/git/config.py +++ b/git/config.py @@ -452,7 +452,7 @@ def _read(self, fp: Union[BufferedReader, IO[bytes]], fpname: str) -> None: e = None # None, or an exception. def string_decode(v: str) -> str: - if v[-1] == "\\": + if v and v.endswith("\\"): v = v[:-1] # END cut trailing escapes to prevent decode error diff --git a/git/diff.py b/git/diff.py index f89b12d98..9c6ae59e0 100644 --- a/git/diff.py +++ b/git/diff.py @@ -187,7 +187,7 @@ def diff( paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, - ) -> "DiffIndex": + ) -> "DiffIndex[Diff]": """Create diffs between two items being trees, trees and index or an index and the working tree. Detects renames automatically. @@ -325,7 +325,7 @@ def iter_change_type(self, change_type: Lit_change_type) -> Iterator[T_Diff]: yield diffidx elif change_type == "C" and diffidx.copied_file: yield diffidx - elif change_type == "R" and diffidx.renamed: + elif change_type == "R" and diffidx.renamed_file: yield diffidx elif change_type == "M" and diffidx.a_blob and diffidx.b_blob and diffidx.a_blob != diffidx.b_blob: yield diffidx @@ -581,7 +581,7 @@ def _pick_best_path(cls, path_match: bytes, rename_match: bytes, path_fallback_m return None @classmethod - def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex: + def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoInterrupt"]) -> DiffIndex["Diff"]: """Create a new :class:`DiffIndex` from the given process output which must be in patch format. @@ -674,7 +674,7 @@ def _index_from_patch_format(cls, repo: "Repo", proc: Union["Popen", "Git.AutoIn return index @staticmethod - def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> None: + def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]) -> None: lines = lines_bytes.decode(defenc) # Discard everything before the first colon, and the colon itself. @@ -695,7 +695,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non change_type: Lit_change_type = cast(Lit_change_type, _change_type[0]) score_str = "".join(_change_type[1:]) score = int(score_str) if score_str.isdigit() else None - path = path.strip() + path = path.strip("\n") a_path = path.encode(defenc) b_path = path.encode(defenc) deleted_file = False @@ -747,7 +747,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex) -> Non index.append(diff) @classmethod - def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex": + def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex[Diff]": """Create a new :class:`DiffIndex` from the given process output which must be in raw format. diff --git a/git/ext/gitdb b/git/ext/gitdb index 3d3e9572d..775cfe829 160000 --- a/git/ext/gitdb +++ b/git/ext/gitdb @@ -1 +1 @@ -Subproject commit 3d3e9572dc452fea53d328c101b3d1440bbefe40 +Subproject commit 775cfe8299ea5474f605935469359a9d1cdb49dc diff --git a/git/index/base.py b/git/index/base.py index b8161ea52..39cc9143c 100644 --- a/git/index/base.py +++ b/git/index/base.py @@ -28,6 +28,7 @@ from git.objects import Blob, Commit, Object, Submodule, Tree from git.objects.util import Serializable from git.util import ( + Actor, LazyMixin, LockedFD, join_path_native, @@ -76,7 +77,6 @@ from git.refs.reference import Reference from git.repo import Repo - from git.util import Actor Treeish = Union[Tree, Commit, str, bytes] @@ -653,12 +653,12 @@ def _to_relative_path(self, path: PathLike) -> PathLike: return path if self.repo.bare: raise InvalidGitRepositoryError("require non-bare repository") - if not str(path).startswith(str(self.repo.working_tree_dir)): + if not osp.normpath(str(path)).startswith(str(self.repo.working_tree_dir)): raise ValueError("Absolute path %r is not in git repository at %r" % (path, self.repo.working_tree_dir)) return os.path.relpath(path, self.repo.working_tree_dir) def _preprocess_add_items( - self, items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]] + self, items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]] ) -> Tuple[List[PathLike], List[BaseIndexEntry]]: """Split the items into two lists of path strings and BaseEntries.""" paths = [] @@ -749,7 +749,7 @@ def _entries_for_paths( def add( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], force: bool = True, fprogress: Callable = lambda *args: None, path_rewriter: Union[Callable[..., PathLike], None] = None, @@ -976,7 +976,7 @@ def _items_to_rela_paths( @default_index def remove( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], working_tree: bool = False, **kwargs: Any, ) -> List[str]: @@ -1036,7 +1036,7 @@ def remove( @default_index def move( self, - items: Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]], + items: Union[PathLike, Sequence[Union[PathLike, Blob, BaseIndexEntry, "Submodule"]]], skip_errors: bool = False, **kwargs: Any, ) -> List[Tuple[str, str]]: @@ -1117,8 +1117,8 @@ def commit( message: str, parent_commits: Union[List[Commit], None] = None, head: bool = True, - author: Union[None, "Actor"] = None, - committer: Union[None, "Actor"] = None, + author: Union[None, Actor] = None, + committer: Union[None, Actor] = None, author_date: Union[datetime.datetime, str, None] = None, commit_date: Union[datetime.datetime, str, None] = None, skip_hooks: bool = False, @@ -1443,7 +1443,7 @@ def reset( key = entry_key(path, 0) self.entries[key] = nie[key] except KeyError: - # If key is not in theirs, it musn't be in ours. + # If key is not in theirs, it mustn't be in ours. try: del self.entries[key] except KeyError: @@ -1478,7 +1478,7 @@ def diff( paths: Union[PathLike, List[PathLike], Tuple[PathLike, ...], None] = None, create_patch: bool = False, **kwargs: Any, - ) -> git_diff.DiffIndex: + ) -> git_diff.DiffIndex[git_diff.Diff]: """Diff this index against the working copy or a :class:`~git.objects.tree.Tree` or :class:`~git.objects.commit.Commit` object. diff --git a/git/objects/commit.py b/git/objects/commit.py index d957c9051..0ceb46609 100644 --- a/git/objects/commit.py +++ b/git/objects/commit.py @@ -377,15 +377,25 @@ def stats(self) -> Stats: :return: :class:`Stats` """ - if not self.parents: - text = self.repo.git.diff_tree(self.hexsha, "--", numstat=True, no_renames=True, root=True) - text2 = "" - for line in text.splitlines()[1:]: + + def process_lines(lines: List[str]) -> str: + text = "" + for file_info, line in zip(lines, lines[len(lines) // 2 :]): + change_type = file_info.split("\t")[0][-1] (insertions, deletions, filename) = line.split("\t") - text2 += "%s\t%s\t%s\n" % (insertions, deletions, filename) - text = text2 + text += "%s\t%s\t%s\t%s\n" % (change_type, insertions, deletions, filename) + return text + + if not self.parents: + lines = self.repo.git.diff_tree( + self.hexsha, "--", numstat=True, no_renames=True, root=True, raw=True + ).splitlines()[1:] + text = process_lines(lines) else: - text = self.repo.git.diff(self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True) + lines = self.repo.git.diff( + self.parents[0].hexsha, self.hexsha, "--", numstat=True, no_renames=True, raw=True + ).splitlines() + text = process_lines(lines) return Stats._list_from_string(self.repo, text) @property diff --git a/git/objects/tag.py b/git/objects/tag.py index a3bb0b882..88671d316 100644 --- a/git/objects/tag.py +++ b/git/objects/tag.py @@ -14,7 +14,7 @@ import sys from git.compat import defenc -from git.util import hex_to_bin +from git.util import Actor, hex_to_bin from . import base from .util import get_object_type_by_name, parse_actor_and_date @@ -30,7 +30,6 @@ if TYPE_CHECKING: from git.repo import Repo - from git.util import Actor from .blob import Blob from .commit import Commit @@ -64,7 +63,7 @@ def __init__( binsha: bytes, object: Union[None, base.Object] = None, tag: Union[None, str] = None, - tagger: Union[None, "Actor"] = None, + tagger: Union[None, Actor] = None, tagged_date: Union[int, None] = None, tagger_tz_offset: Union[int, None] = None, message: Union[str, None] = None, diff --git a/git/objects/util.py b/git/objects/util.py index 5c56e6134..a68d701f5 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -568,11 +568,11 @@ def addToStack( yield rval # Only continue to next level if this is appropriate! - nd = d + 1 - if depth > -1 and nd > depth: + next_d = d + 1 + if depth > -1 and next_d > depth: continue - addToStack(stack, item, branch_first, nd) + addToStack(stack, item, branch_first, next_d) # END for each item on work stack diff --git a/git/refs/head.py b/git/refs/head.py index 7e6fc3377..683634451 100644 --- a/git/refs/head.py +++ b/git/refs/head.py @@ -99,8 +99,8 @@ def reset( if index: mode = "--mixed" - # It appears some git versions declare mixed and paths deprecated. - # See http://github.com/Byron/GitPython/issues#issue/2. + # Explicit "--mixed" when passing paths is deprecated since git 1.5.4. + # See https://github.com/gitpython-developers/GitPython/discussions/1876. if paths: mode = None # END special case diff --git a/git/remote.py b/git/remote.py index f2ecd0f36..20e42b412 100644 --- a/git/remote.py +++ b/git/remote.py @@ -250,7 +250,7 @@ def _from_line(cls, remote: "Remote", line: str) -> "PushInfo": flags |= cls.NEW_TAG elif "[new branch]" in summary: flags |= cls.NEW_HEAD - # uptodate encoded in control character + # `uptodate` encoded in control character else: # Fast-forward or forced update - was encoded in control character, # but we parse the old and new commit. @@ -316,7 +316,7 @@ class FetchInfo(IterableObj): ERROR, ) = [1 << x for x in range(8)] - _re_fetch_result = re.compile(r"^ *(.) (\[[\w \.$@]+\]|[\w\.$@]+) +(.+) -> ([^ ]+)( \(.*\)?$)?") + _re_fetch_result = re.compile(r"^ *(?:.{0,3})(.) (\[[\w \.$@]+\]|[\w\.$@]+) +(.+) -> ([^ ]+)( \(.*\)?$)?") _flag_map: Dict[flagKeyLiteral, int] = { "!": ERROR, @@ -828,8 +828,15 @@ def remove(cls, repo: "Repo", name: str) -> str: name._clear_cache() return name - # `rm` is an alias. - rm = remove + @classmethod + def rm(cls, repo: "Repo", name: str) -> str: + """Alias of remove. + Remove the remote with the given name. + + :return: + The passed remote name to remove + """ + return cls.remove(repo, name) def rename(self, new_name: str) -> "Remote": """Rename self to the given `new_name`. @@ -887,7 +894,7 @@ def _get_fetch_info_from_stderr( None, progress_handler, finalizer=None, - decode_streams=True, + decode_streams=False, kill_after_timeout=kill_after_timeout, ) @@ -1064,7 +1071,7 @@ def fetch( Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_fetch_options) proc = self.repo.git.fetch( - "--", self, *args, as_process=True, with_stdout=False, universal_newlines=False, v=verbose, **kwargs + "--", self, *args, as_process=True, with_stdout=False, universal_newlines=True, v=verbose, **kwargs ) res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout) if hasattr(self.repo.odb, "update_cache"): @@ -1118,7 +1125,7 @@ def pull( Git.check_unsafe_options(options=list(kwargs.keys()), unsafe_options=self.unsafe_git_pull_options) proc = self.repo.git.pull( - "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=False, v=True, **kwargs + "--", self, refspec, with_stdout=False, as_process=True, universal_newlines=True, v=True, **kwargs ) res = self._get_fetch_info_from_stderr(proc, progress, kill_after_timeout=kill_after_timeout) if hasattr(self.repo.odb, "update_cache"): diff --git a/git/repo/base.py b/git/repo/base.py index 51ea76901..db89cdf41 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -179,7 +179,7 @@ def __init__( R"""Create a new :class:`Repo` instance. :param path: - The path to either the root git directory or the bare git repo:: + The path to either the worktree directory or the .git directory itself:: repo = Repo("/Users/mtrier/Development/git-python") repo = Repo("/Users/mtrier/Development/git-python.git") @@ -402,6 +402,17 @@ def heads(self) -> "IterableList[Head]": """ return Head.list_items(self) + @property + def branches(self) -> "IterableList[Head]": + """Alias for heads. + A list of :class:`~git.refs.head.Head` objects representing the branch heads + in this repo. + + :return: + ``git.IterableList(Head, ...)`` + """ + return self.heads + @property def references(self) -> "IterableList[Reference]": """A list of :class:`~git.refs.reference.Reference` objects representing tags, @@ -412,11 +423,16 @@ def references(self) -> "IterableList[Reference]": """ return Reference.list_items(self) - # Alias for references. - refs = references + @property + def refs(self) -> "IterableList[Reference]": + """Alias for references. + A list of :class:`~git.refs.reference.Reference` objects representing tags, + heads and remote references. - # Alias for heads. - branches = heads + :return: + ``git.IterableList(Reference, ...)`` + """ + return self.references @property def index(self) -> "IndexFile": diff --git a/git/repo/fun.py b/git/repo/fun.py index e44d9c644..182cf82ed 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -112,7 +112,7 @@ def find_submodule_git_dir(d: PathLike) -> Optional[PathLike]: path = content[8:] if Git.is_cygwin(): - # Cygwin creates submodules prefixed with `/cygdrive/...` suffixes. + # Cygwin creates submodules prefixed with `/cygdrive/...`. # Cygwin git understands Cygwin paths much better than Windows ones. # Also the Cygwin tests are assuming Cygwin paths. path = cygpath(path) diff --git a/git/types.py b/git/types.py index 584450146..cce184530 100644 --- a/git/types.py +++ b/git/types.py @@ -248,6 +248,7 @@ class Files_TD(TypedDict): insertions: int deletions: int lines: int + change_type: str class Total_TD(TypedDict): diff --git a/git/util.py b/git/util.py index 8c1c26012..9e8ac821d 100644 --- a/git/util.py +++ b/git/util.py @@ -339,7 +339,7 @@ def _get_exe_extensions() -> Sequence[str]: if PATHEXT: return tuple(p.upper() for p in PATHEXT.split(os.pathsep)) elif sys.platform == "win32": - return (".BAT", "COM", ".EXE") + return (".BAT", ".COM", ".EXE") else: return () @@ -910,6 +910,7 @@ class Stats: deletions = number of deleted lines as int insertions = number of inserted lines as int lines = total number of lines changed as int, or deletions + insertions + change_type = type of change as str, A|C|D|M|R|T|U|X|B ``full-stat-dict`` @@ -938,7 +939,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": "files": {}, } for line in text.splitlines(): - (raw_insertions, raw_deletions, filename) = line.split("\t") + (change_type, raw_insertions, raw_deletions, filename) = line.split("\t") insertions = raw_insertions != "-" and int(raw_insertions) or 0 deletions = raw_deletions != "-" and int(raw_deletions) or 0 hsh["total"]["insertions"] += insertions @@ -949,6 +950,7 @@ def _list_from_string(cls, repo: "Repo", text: str) -> "Stats": "insertions": insertions, "deletions": deletions, "lines": insertions + deletions, + "change_type": change_type, } hsh["files"][filename.strip()] = files_dict return Stats(hsh["total"], hsh["files"]) diff --git a/init-tests-after-clone.sh b/init-tests-after-clone.sh index 118e1de22..bfada01b0 100755 --- a/init-tests-after-clone.sh +++ b/init-tests-after-clone.sh @@ -1,4 +1,7 @@ #!/bin/sh +# +# This file is part of GitPython and is released under the +# 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/ set -eu diff --git a/pyproject.toml b/pyproject.toml index 6cb05f96e..090972eed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -addopts = "--cov=git --cov-report=term --disable-warnings -ra" +addopts = "--cov=git --cov-report=term -ra" filterwarnings = "ignore::DeprecationWarning" python_files = "test_*.py" tmp_path_retention_policy = "failed" @@ -13,7 +13,6 @@ testpaths = "test" # Space separated list of paths from root e.g test tests doc # --cov-report term-missing # to terminal with line numbers # --cov-report html:path # html file at path # --maxfail # number of errors before giving up -# -disable-warnings # Disable pytest warnings (not codebase warnings) # -rfE # default test summary: list fail and error # -ra # test summary: list all non-passing (fail, error, skip, xfail, xpass) # --ignore-glob=**/gitdb/* # ignore glob paths @@ -79,3 +78,12 @@ lint.unfixable = [ "test/**" = [ "B018", # useless-expression ] +"fuzzing/fuzz-targets/**" = [ + "E402", # environment setup must happen before the `git` module is imported, thus cannot happen at top of file +] + + +[tool.codespell] +ignore-words-list="afile,assertIn,doesnt,gud,uptodate" +#count = true +quiet-level = 3 diff --git a/test/deprecation/test_basic.py b/test/deprecation/test_basic.py index 6235a836c..3bf0287c7 100644 --- a/test/deprecation/test_basic.py +++ b/test/deprecation/test_basic.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: from pathlib import Path - from git.diff import Diff + from git.diff import Diff, DiffIndex from git.objects.commit import Commit # ------------------------------------------------------------------------ @@ -54,6 +54,12 @@ def diff(commit: "Commit") -> Generator["Diff", None, None]: yield diff +@pytest.fixture +def diffs(commit: "Commit") -> Generator["DiffIndex", None, None]: + """Fixture to supply a DiffIndex.""" + yield commit.diff(NULL_TREE) + + def test_diff_renamed_warns(diff: "Diff") -> None: """The deprecated Diff.renamed property issues a deprecation warning.""" with pytest.deprecated_call(): @@ -122,3 +128,10 @@ def test_iterable_obj_inheriting_does_not_warn() -> None: class Derived(IterableObj): pass + + +def test_diff_iter_change_type(diffs: "DiffIndex") -> None: + """The internal DiffIndex.iter_change_type function issues no deprecation warning.""" + with assert_no_deprecation_warning(): + for change_type in diffs.change_type: + [*diffs.iter_change_type(change_type=change_type)] diff --git a/test/fixtures/diff_numstat b/test/fixtures/diff_numstat index 44c6ca2d5..b76e467eb 100644 --- a/test/fixtures/diff_numstat +++ b/test/fixtures/diff_numstat @@ -1,2 +1,3 @@ -29 18 a.txt -0 5 b.txt +M 29 18 a.txt +M 0 5 b.txt +A 7 0 c.txt \ No newline at end of file diff --git a/test/test_commit.py b/test/test_commit.py index 5832258de..37c66e3e7 100644 --- a/test/test_commit.py +++ b/test/test_commit.py @@ -135,9 +135,12 @@ def test_stats(self): commit = self.rorepo.commit("33ebe7acec14b25c5f84f35a664803fcab2f7781") stats = commit.stats - def check_entries(d): + def check_entries(d, has_change_type=False): assert isinstance(d, dict) - for key in ("insertions", "deletions", "lines"): + keys = ("insertions", "deletions", "lines") + if has_change_type: + keys += ("change_type",) + for key in keys: assert key in d # END assertion helper @@ -148,7 +151,7 @@ def check_entries(d): assert "files" in stats.total for _filepath, d in stats.files.items(): - check_entries(d) + check_entries(d, True) # END for each stated file # Check that data is parsed properly. diff --git a/test/test_config.py b/test/test_config.py index 0911d0262..92997422d 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -142,6 +142,14 @@ def test_multi_line_config(self): ) self.assertEqual(len(config.sections()), 23) + def test_config_value_with_trailing_new_line(self): + config_content = b'[section-header]\nkey:"value\n"' + config_file = io.BytesIO(config_content) + config_file.name = "multiline_value.config" + + git_config = GitConfigParser(config_file) + git_config.read() # This should not throw an exception + def test_base(self): path_repo = fixture_path("git_config") path_global = fixture_path("git_config_global") diff --git a/test/test_diff.py b/test/test_diff.py index 928a9f428..612fbd9e0 100644 --- a/test/test_diff.py +++ b/test/test_diff.py @@ -529,3 +529,22 @@ def test_diff_patch_with_external_engine(self, rw_dir): self.assertEqual(len(index_against_head), 1) index_against_working_tree = repo.index.diff(None, create_patch=True) self.assertEqual(len(index_against_working_tree), 1) + + @with_rw_directory + def test_beginning_space(self, rw_dir): + # Create a file beginning by a whitespace + repo = Repo.init(rw_dir) + file = osp.join(rw_dir, " file.txt") + with open(file, "w") as f: + f.write("hello world") + repo.git.add(Git.polish_url(https://github.com/file)) + repo.index.commit("first commit") + + # Diff the commit with an empty tree + # and check the paths + diff_index = repo.head.commit.diff(NULL_TREE) + d = diff_index[0] + a_path = d.a_path + b_path = d.b_path + self.assertEqual(a_path, " file.txt") + self.assertEqual(b_path, " file.txt") diff --git a/test/test_docs.py b/test/test_docs.py index b3547c1de..cc0bbf26a 100644 --- a/test/test_docs.py +++ b/test/test_docs.py @@ -469,11 +469,11 @@ def test_references_and_objects(self, rw_dir): # ![30-test_references_and_objects] # [31-test_references_and_objects] - git = repo.git - git.checkout("HEAD", b="my_new_branch") # Create a new branch. - git.branch("another-new-one") - git.branch("-D", "another-new-one") # Pass strings for full control over argument order. - git.for_each_ref() # '-' becomes '_' when calling it. + git_cmd = repo.git + git_cmd.checkout("HEAD", b="my_new_branch") # Create a new branch. + git_cmd.branch("another-new-one") + git_cmd.branch("-D", "another-new-one") # Pass strings for full control over argument order. + git_cmd.for_each_ref() # '-' becomes '_' when calling it. # ![31-test_references_and_objects] repo.git.clear_cache() diff --git a/test/test_exc.py b/test/test_exc.py index c1eae7240..2e979f5a1 100644 --- a/test/test_exc.py +++ b/test/test_exc.py @@ -52,7 +52,7 @@ _streams_n_substrings = ( None, - "steram", + "stream", "ομορφο stream", ) diff --git a/test/test_index.py b/test/test_index.py index b92258c92..c586a0b5a 100644 --- a/test/test_index.py +++ b/test/test_index.py @@ -1018,7 +1018,7 @@ class Mocked: @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, @@ -1077,7 +1077,7 @@ def test_hook_uses_shell_not_from_cwd(self, rw_dir, case): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.WslNoDistro, @@ -1120,7 +1120,7 @@ def test_pre_commit_hook_fail(self, rw_repo): @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Absent, reason="Can't run a hook on Windows without bash.exe.", - rasies=HookExecutionError, + raises=HookExecutionError, ) @pytest.mark.xfail( type(_win_bash_status) is WinBashStatus.Wsl, @@ -1181,6 +1181,18 @@ def test_index_add_pathlike(self, rw_repo): rw_repo.index.add(file) + @with_rw_repo("HEAD") + def test_index_add_non_normalized_path(self, rw_repo): + git_dir = Path(rw_repo.git_dir) + + file = git_dir / "file.txt" + file.touch() + non_normalized_path = file.as_posix() + if os.name != "nt": + non_normalized_path = "/" + non_normalized_path[1:].replace("/", "//") + + rw_repo.index.add(non_normalized_path) + class TestIndexUtils: @pytest.mark.parametrize("file_path_type", [str, Path]) diff --git a/test/test_stats.py b/test/test_stats.py index eec73c802..91d2cf6ae 100644 --- a/test/test_stats.py +++ b/test/test_stats.py @@ -14,13 +14,19 @@ def test_list_from_string(self): output = fixture("diff_numstat").decode(defenc) stats = Stats._list_from_string(self.rorepo, output) - self.assertEqual(2, stats.total["files"]) - self.assertEqual(52, stats.total["lines"]) - self.assertEqual(29, stats.total["insertions"]) + self.assertEqual(3, stats.total["files"]) + self.assertEqual(59, stats.total["lines"]) + self.assertEqual(36, stats.total["insertions"]) self.assertEqual(23, stats.total["deletions"]) self.assertEqual(29, stats.files["a.txt"]["insertions"]) self.assertEqual(18, stats.files["a.txt"]["deletions"]) + self.assertEqual("M", stats.files["a.txt"]["change_type"]) self.assertEqual(0, stats.files["b.txt"]["insertions"]) self.assertEqual(5, stats.files["b.txt"]["deletions"]) + self.assertEqual("M", stats.files["b.txt"]["change_type"]) + + self.assertEqual(7, stats.files["c.txt"]["insertions"]) + self.assertEqual(0, stats.files["c.txt"]["deletions"]) + self.assertEqual("A", stats.files["c.txt"]["change_type"])