From 7ab12b403207bb46199f46d5aaa72d3e82a3080d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 29 Jul 2015 18:29:03 +0200 Subject: [PATCH 001/885] fix(index):allow adding non-unicode paths to index This issue only surfaced in python 2, in case paths containing unicode characters were not actual unicode objects. In python 3, this was never the issue. Closes #331 --- git/index/fun.py | 5 +++-- git/test/test_index.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/git/index/fun.py b/git/index/fun.py index 9ae468611..c1026fd62 100644 --- a/git/index/fun.py +++ b/git/index/fun.py @@ -41,7 +41,8 @@ from gitdb.typ import str_tree_type from git.compat import ( defenc, - force_text + force_text, + force_bytes ) S_IFGITLINK = S_IFLNK | S_IFDIR # a submodule @@ -124,7 +125,7 @@ def write_cache(entries, stream, extension_data=None, ShaStreamCls=IndexFileSHA1 write(entry[4]) # ctime write(entry[5]) # mtime path = entry[3] - path = path.encode(defenc) + path = force_bytes(path, encoding=defenc) plen = len(path) & CE_NAMEMASK # path length assert plen == len(path), "Path %s too long to fit into index" % entry[3] flags = plen | (entry[2] & CE_NAMEMASK_INV) # clear possible previous values diff --git a/git/test/test_index.py b/git/test/test_index.py index 2fd53f656..397885752 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -18,6 +18,7 @@ ) from git import ( IndexFile, + Repo, BlobFilter, UnmergedEntriesError, Tree, @@ -45,6 +46,7 @@ IndexEntry ) from git.index.fun import hook_path +from gitdb.test.lib import with_rw_directory class TestIndex(TestBase): @@ -780,3 +782,14 @@ def test_index_bare_add(self, rw_bare_repo): except InvalidGitRepositoryError: asserted = True assert asserted, "Adding using a filename is not correctly asserted." + + @with_rw_directory + def test_add_utf8P_path(self, rw_dir): + # NOTE: fp is not a Unicode object in python 2 (which is the source of the problem) + fp = os.path.join(rw_dir, 'ø.txt') + with open(fp, 'w') as fs: + fs.write('content of ø') + + r = Repo.init(rw_dir) + r.index.add([fp]) + r.index.commit('Added orig and prestable') From b2e4134963c971e3259137b84237d6c47964b018 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 8 Aug 2015 22:25:09 +0200 Subject: [PATCH 002/885] docs(README): removed coveralls link Coveralls was disabled a while ago because it would prevent merging PRs from the phone. --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index f87cb0efe..75fcdcb2c 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,6 @@ New BSD License. See the LICENSE file. [![Build Status](https://travis-ci.org/gitpython-developers/GitPython.svg)](https://travis-ci.org/gitpython-developers/GitPython) [![Code Climate](https://codeclimate.com/github/gitpython-developers/GitPython/badges/gpa.svg)](https://codeclimate.com/github/gitpython-developers/GitPython) -[![Coverage Status](https://coveralls.io/repos/gitpython-developers/GitPython/badge.png?branch=master)](https://coveralls.io/r/gitpython-developers/GitPython?branch=master) [![Documentation Status](https://readthedocs.org/projects/gitpython/badge/?version=stable)](https://readthedocs.org/projects/gitpython/?badge=stable) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/GitPython/badge/pr)](http://www.issuestats.com/github/gitpython-developers/GitPython) [![Issue Stats](http://www.issuestats.com/github/gitpython-developers/GitPython/badge/issue)](http://www.issuestats.com/github/gitpython-developers/GitPython) From a0acb2229e1ebb59ab343e266fc5c1cc392a974e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 8 Aug 2015 23:02:38 +0200 Subject: [PATCH 003/885] tests(submodule): add new submodule commits It's somewhat more complex to add new commits in submodules to the parent repository. The new test shows how to do that in a 'GitPythonic' way. Related to #335 --- git/test/test_submodule.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/git/test/test_submodule.py b/git/test/test_submodule.py index cbf38c182..3d78d0672 100644 --- a/git/test/test_submodule.py +++ b/git/test/test_submodule.py @@ -661,7 +661,7 @@ def test_add_empty_repo(self, rwdir): # end for each checkout mode @with_rw_directory - def test_git_submodules(self, rwdir): + def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): parent = git.Repo.init(os.path.join(rwdir, 'parent')) parent.git.submodule('add', self._small_repo_url(), 'module') parent.index.commit("added submodule") @@ -686,6 +686,37 @@ def test_git_submodules(self, rwdir): sm.move(sm.path + '_moved') sm2.move(sm2.path + '_moved') + parent.index.commit("moved submodules") + + smm = sm.module() + fp = os.path.join(smm.working_tree_dir, 'empty-file') + with open(fp, 'w'): + pass + smm.git.add(fp) + smm.git.commit(m="new file added") + + # submodules are retrieved from the current commit's tree, therefore we can't really get a new submodule + # object pointing to the new submodule commit + sm_too = parent.submodules[0] + assert parent.head.commit.tree[sm.path].binsha == sm.binsha + assert sm_too.binsha == sm.binsha, "cached submodule should point to the same commit as updated one" + + added_bies = parent.index.add([sm]) # addded base-index-entries + assert len(added_bies) == 1 + parent.index.commit("add same submodule entry") + commit_sm = parent.head.commit.tree[sm.path] + assert commit_sm.binsha == added_bies[0].binsha + assert commit_sm.binsha == sm.binsha + + sm_too.binsha = sm_too.module().head.commit.binsha + added_bies = parent.index.add([sm_too]) + assert len(added_bies) == 1 + parent.index.commit("add new submodule entry") + commit_sm = parent.head.commit.tree[sm.path] + assert commit_sm.binsha == added_bies[0].binsha + assert commit_sm.binsha == sm_too.binsha + assert sm_too.binsha != sm.binsha + @with_rw_directory def test_git_submodule_compatibility(self, rwdir): parent = git.Repo.init(os.path.join(rwdir, 'parent')) From 039e265819cc6e5241907f1be30d2510bfa5ca6c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 8 Aug 2015 23:35:12 +0200 Subject: [PATCH 004/885] fix(tests): remove dependency on sort order Now we select the submodule by name, not by index. The latter is not deterministic. Closes #335 --- git/test/test_submodule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git/test/test_submodule.py b/git/test/test_submodule.py index 3d78d0672..17ce605a4 100644 --- a/git/test/test_submodule.py +++ b/git/test/test_submodule.py @@ -697,7 +697,7 @@ def test_git_submodules_and_add_sm_with_new_commit(self, rwdir): # submodules are retrieved from the current commit's tree, therefore we can't really get a new submodule # object pointing to the new submodule commit - sm_too = parent.submodules[0] + sm_too = parent.submodules['module_moved'] assert parent.head.commit.tree[sm.path].binsha == sm.binsha assert sm_too.binsha == sm.binsha, "cached submodule should point to the same commit as updated one" From a8f7e3772f68c8e6350b9ff5ac981ba3223f2d43 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 17 Aug 2015 22:32:15 +0200 Subject: [PATCH 005/885] fix(commit): serialization timezone handling Previously timezones which were not divisable by 3600s would be parsed correctly, but would serialize into a full hour, rounded up. Now floating point computation is used which fixes the issue. Related to #336 --- doc/source/changes.rst | 7 +++++++ git/objects/util.py | 2 +- git/test/performance/test_commit.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index e6d7b09be..970ba1958 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -2,6 +2,13 @@ Changelog ========= +1.0.2 - Fixes +============= + +* CRITICAL: fixed incorrect `Commit` object serialization when authored or commit date had timezones which were not + divisable by 3600 seconds. This would happen if the timezone was something like `+0530` for instance. +* A list of all additional fixes can be found `on github `_ + 1.0.1 - Fixes ============= diff --git a/git/objects/util.py b/git/objects/util.py index 567b1d5b2..8fd92a0a9 100644 --- a/git/objects/util.py +++ b/git/objects/util.py @@ -73,7 +73,7 @@ def utctz_to_altz(utctz): def altz_to_utctz_str(altz): """As above, but inverses the operation, returning a string that can be used in commit objects""" - utci = -1 * int((altz / 3600) * 100) + utci = -1 * int((float(altz) / 3600) * 100) utcs = str(abs(utci)) utcs = "0" * (4 - len(utcs)) + utcs prefix = (utci < 0 and '-') or '+' diff --git a/git/test/performance/test_commit.py b/git/test/performance/test_commit.py index 7d3e87c4c..b59c747ee 100644 --- a/git/test/performance/test_commit.py +++ b/git/test/performance/test_commit.py @@ -76,7 +76,7 @@ def test_commit_iteration(self): % (nc, elapsed_time, nc / elapsed_time), file=sys.stderr) def test_commit_serialization(self): - assert_commit_serialization(self.gitrwrepo, self.gitrwrepo.head, True) + assert_commit_serialization(self.gitrwrepo, '58c78e6', True) rwrepo = self.gitrwrepo make_object = rwrepo.odb.store From c8b837923d506e265ff8bb79af61c0d86e7d5b2e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 17 Aug 2015 22:39:13 +0200 Subject: [PATCH 006/885] fix(test_index): fix encoding I really never want to touch python again, and never deal with py2/3 unicode handling anymore. By now, it seems pretty much anything is better. Is python to be blamed for it entirely ? Probably not, but there are better alternatives. Nim ? Rust ? Ruby ? Totally --- git/test/test_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git/test/test_index.py b/git/test/test_index.py index 397885752..ffc4bffe1 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -787,8 +787,8 @@ def test_index_bare_add(self, rw_bare_repo): def test_add_utf8P_path(self, rw_dir): # NOTE: fp is not a Unicode object in python 2 (which is the source of the problem) fp = os.path.join(rw_dir, 'ø.txt') - with open(fp, 'w') as fs: - fs.write('content of ø') + with open(fp, 'wb') as fs: + fs.write(u'content of ø'.encode('utf-8')) r = Repo.init(rw_dir) r.index.add([fp]) From 6eb3af27464ffba83e3478b0a0c8b1f9ff190889 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 22 Aug 2015 16:31:31 +0200 Subject: [PATCH 007/885] fix(repo): use GitCmdObjectDB by default This should fix resource leaking issues once and for all. Related #304 --- doc/source/changes.rst | 3 +++ git/repo/base.py | 7 ++----- git/test/test_repo.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 970ba1958..acf681eea 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -5,6 +5,9 @@ Changelog 1.0.2 - Fixes ============= +* IMPORTANT: Changed default object database of `Repo` objects to `GitComdObjectDB`. The pure-python implementation + used previously usually fails to release its resources (i.e. file handles), which can lead to problems when working + with large repositories. * CRITICAL: fixed incorrect `Commit` object serialization when authored or commit date had timezones which were not divisable by 3600 seconds. This would happen if the timezone was something like `+0530` for instance. * A list of all additional fixes can be found `on github `_ diff --git a/git/repo/base.py b/git/repo/base.py index 16fb58e59..d5bc24d87 100644 --- a/git/repo/base.py +++ b/git/repo/base.py @@ -35,10 +35,7 @@ add_progress ) -from git.db import ( - GitCmdObjectDB, - GitDB -) +from git.db import GitCmdObjectDB from gitdb.util import ( join, @@ -62,7 +59,7 @@ import sys import re -DefaultDBType = GitDB +DefaultDBType = GitCmdObjectDB if sys.version_info[:2] < (2, 5): # python 2.4 compatiblity DefaultDBType = GitCmdObjectDB # END handle python 2.4 diff --git a/git/test/test_repo.py b/git/test/test_repo.py index bc9f3e92c..c95592ea9 100644 --- a/git/test/test_repo.py +++ b/git/test/test_repo.py @@ -657,7 +657,7 @@ def test_rev_parse(self): assert rev_parse('@{1}') != head.commit def test_repo_odbtype(self): - target_type = GitDB + target_type = GitCmdObjectDB if sys.version_info[:2] < (2, 5): target_type = GitCmdObjectDB assert isinstance(self.rorepo.odb, target_type) From e8590424997ab1d578c777fe44bf7e4173036f93 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 29 Aug 2015 16:19:52 +0200 Subject: [PATCH 008/885] fix(repo): fail loudly if worktrees are used As GitPython is in maintenance mode, there will be no new features. However, I believe it's good idea to explicitly state we do not support certain things if this is the case. Therefore, when worktrees are encountered, we will throw an specific exception to indicate that. The current implementation is hacky to speed up development, and increases the risk of failing due to false-positive worktree directories. Related to #344 --- git/exc.py | 4 ++++ git/repo/fun.py | 25 ++++++++++++++++--------- git/test/test_repo.py | 20 +++++++++++++++++++- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/git/exc.py b/git/exc.py index f5b52374b..34382ecd5 100644 --- a/git/exc.py +++ b/git/exc.py @@ -14,6 +14,10 @@ class InvalidGitRepositoryError(Exception): """ Thrown if the given repository appears to have an invalid format. """ +class WorkTreeRepositoryUnsupported(InvalidGitRepositoryError): + """ Thrown to indicate we can't handle work tree repositories """ + + class NoSuchPathError(OSError): """ Thrown if a path could not be access by the system. """ diff --git a/git/repo/fun.py b/git/repo/fun.py index 049800b93..6b06663a0 100644 --- a/git/repo/fun.py +++ b/git/repo/fun.py @@ -4,7 +4,7 @@ from gitdb.exc import ( BadObject, - BadName + BadName, ) from git.refs import SymbolicReference from git.objects import Object @@ -16,6 +16,7 @@ hex_to_bin, bin_to_hex ) +from git.exc import WorkTreeRepositoryUnsupported from git.compat import xrange @@ -31,14 +32,20 @@ def touch(filename): def is_git_dir(d): """ This is taken from the git setup.c:is_git_directory - function.""" - if isdir(d) and \ - isdir(join(d, 'objects')) and \ - isdir(join(d, 'refs')): - headref = join(d, 'HEAD') - return isfile(headref) or \ - (os.path.islink(headref) and - os.readlink(headref).startswith('refs')) + function. + + @throws WorkTreeRepositoryUnsupported if it sees a worktree directory. It's quite hacky to do that here, + but at least clearly indicates that we don't support it. + There is the unlikely danger to throw if we see directories which just look like a worktree dir, + but are none.""" + if isdir(d): + if isdir(join(d, 'objects')) and isdir(join(d, 'refs')): + headref = join(d, 'HEAD') + return isfile(headref) or \ + (os.path.islink(headref) and + os.readlink(headref).startswith('refs')) + elif isfile(join(d, 'gitdir')) and isfile(join(d, 'commondir')) and isfile(join(d, 'gitfile')): + raise WorkTreeRepositoryUnsupported(d) return False diff --git a/git/test/test_repo.py b/git/test/test_repo.py index c95592ea9..2e44f0aad 100644 --- a/git/test/test_repo.py +++ b/git/test/test_repo.py @@ -33,7 +33,10 @@ ) from git.repo.fun import touch from git.util import join_path_native -from git.exc import BadObject +from git.exc import ( + BadObject, + WorkTreeRepositoryUnsupported +) from gitdb.util import bin_to_hex from git.compat import string_types from gitdb.test.lib import with_rw_directory @@ -45,6 +48,8 @@ import itertools from io import BytesIO +from nose import SkipTest + class TestRepo(TestBase): @@ -779,3 +784,16 @@ def test_is_ancestor(self): self.assertFalse(repo.is_ancestor("master", c1)) for i, j in itertools.permutations([c1, 'ffffff', ''], r=2): self.assertRaises(GitCommandError, repo.is_ancestor, i, j) + + @with_rw_directory + def test_work_tree_unsupported(self, rw_dir): + git = Git(rw_dir) + if git.version_info[:3] < (2, 5, 1): + raise SkipTest("worktree feature unsupported") + + rw_master = self.rorepo.clone(join_path_native(rw_dir, 'master_repo')) + rw_master.git.checkout('HEAD~10') + worktree_path = join_path_native(rw_dir, 'worktree_repo') + rw_master.git.worktree('add', worktree_path, 'master') + + self.failUnlessRaises(WorkTreeRepositoryUnsupported, Repo, worktree_path) From 7f8d9ca08352a28cba3b01e4340a24edc33e13e8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 29 Aug 2015 16:25:40 +0200 Subject: [PATCH 009/885] fix(compat): make test work with git >= 2.5 --- git/test/test_index.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/git/test/test_index.py b/git/test/test_index.py index ffc4bffe1..a928fe5e7 100644 --- a/git/test/test_index.py +++ b/git/test/test_index.py @@ -690,6 +690,9 @@ def make_paths(): index.add(files, write=True) if os.name != 'nt': hp = hook_path('pre-commit', index.repo.git_dir) + hpd = os.path.dirname(hp) + if not os.path.isdir(hpd): + os.mkdir(hpd) with open(hp, "wt") as fp: fp.write("#!/usr/bin/env sh\necho stdout; echo stderr 1>&2; exit 1") # end From 074842accb51b2a0c2c1193018d9f374ac5e948f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 6 Sep 2015 15:11:54 +0200 Subject: [PATCH 010/885] fix(config): ignore empty values in config file Similar to git, we now ignore options which have no value. Previously it would not handle it consistently, and throw a parsing error the first time the cache was built. Afterwards, it was fully usable though. Now we specifically check for the case of no-value options instead. Closes #349 --- git/config.py | 20 +++++++++++++------ git/test/fixtures/git_config_with_empty_value | 4 ++++ git/test/test_config.py | 7 +++++++ 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 git/test/fixtures/git_config_with_empty_value diff --git a/git/config.py b/git/config.py index b7ddf0d22..ea5e17bea 100644 --- a/git/config.py +++ b/git/config.py @@ -156,15 +156,21 @@ class GitConfigParser(with_metaclass(MetaParserBuilder, cp.RawConfigParser, obje #} END configuration + optvalueonly_source = r'\s*(?P