Merge pull request #96 from pre-commit/git_lfs

Teach check-large-files-added about git-lfs.  Reslves #82.

.coveragerc

@@ -7,6 +7,7 @@ omit =
     /usr/*
     */tmp*
     setup.py
+    get-git-lfs.py
 
 [report]
 exclude_lines =

.travis.yml

@@ -12,6 +12,8 @@ script: tox
 before_install:
   - git config --global user.name "Travis CI"
   - git config --global user.email "user@example.com"
+  # Install git-lfs for a test
+  - './get-git-lfs.py && export PATH="/tmp/git-lfs:$PATH"'
 after_success:
     - coveralls
 sudo: false
@@ -19,3 +21,4 @@ cache:
     directories:
         - $HOME/.cache/pip
         - $HOME/.pre-commit
+        - /tmp/git-lfs

get-git-lfs.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3.4
+"""This is a script to install git-lfs to a tempdir for use in tests"""
+import io
+import os.path
+import shutil
+import tarfile
+from urllib.request import urlopen
+
+DOWNLOAD_PATH = (
+    'https://github.com/github/git-lfs/releases/download/'
+    'v1.1.0/git-lfs-linux-amd64-1.1.0.tar.gz'
+)
+PATH_IN_TAR = 'git-lfs-1.1.0/git-lfs'
+DEST_PATH = '/tmp/git-lfs/git-lfs'
+DEST_DIR = os.path.dirname(DEST_PATH)
+
+
+def main():
+    if (
+            os.path.exists(DEST_PATH) and
+            os.path.isfile(DEST_PATH) and
+            os.access(DEST_PATH, os.X_OK)
+    ):
+        print('Already installed!')
+        return 0
+
+    shutil.rmtree(DEST_DIR, ignore_errors=True)
+    os.makedirs(DEST_DIR, exist_ok=True)
+
+    contents = io.BytesIO(urlopen(DOWNLOAD_PATH).read())
+    with tarfile.open(fileobj=contents) as tar:
+        with tar.extractfile(PATH_IN_TAR) as src_file:
+            with open(DEST_PATH, 'wb') as dest_file:
+                shutil.copyfileobj(src_file, dest_file)
+    os.chmod(DEST_PATH, 0o755)
+
+if __name__ == '__main__':
+    exit(main())

pre_commit_hooks/check_added_large_files.py

@@ -9,12 +9,34 @@ import os
 import sys
 
 from pre_commit_hooks.util import added_files
+from pre_commit_hooks.util import CalledProcessError
+from pre_commit_hooks.util import cmd_output
+
+
+def lfs_files():
+    try:  # pragma: no cover (no git-lfs)
+        lines = cmd_output('git', 'lfs', 'status', '--porcelain').splitlines()
+    except CalledProcessError:
+        lines = []
+
+    modes_and_fileparts = [
+        (line[:3].strip(), line[3:].rpartition(' ')[0]) for line in lines
+    ]
+
+    def to_file_part(mode, filepart):
+        assert mode in ('A', 'R')
+        return filepart if mode == 'A' else filepart.split(' -> ')[1]
+
+    return set(
+        to_file_part(mode, filepart) for mode, filepart in modes_and_fileparts
+        if mode in ('A', 'R')
+    )
 
 
 def find_large_added_files(filenames, maxkb):
     # Find all added files that are also in the list of files pre-commit tells
     # us about
-    filenames = added_files() & set(filenames)
+    filenames = (added_files() & set(filenames)) - lfs_files()
 
     retv = 0
     for filename in filenames:

pre_commit_hooks/util.py

@@ -21,7 +21,9 @@ def cmd_output(*cmd, **kwargs):
     popen_kwargs.update(kwargs)
     proc = subprocess.Popen(cmd, **popen_kwargs)
     stdout, stderr = proc.communicate()
-    stdout, stderr = stdout.decode('UTF-8'), stderr.decode('UTF-8')
+    stdout = stdout.decode('UTF-8')
+    if stderr is not None:
+        stderr = stderr.decode('UTF-8')
     if retcode is not None and proc.returncode != retcode:
         raise CalledProcessError(cmd, retcode, proc.returncode, stdout, stderr)
     return stdout

tests/check_added_large_files_test.py

@@ -1,6 +1,10 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import subprocess
+
+import pytest
+
 from pre_commit_hooks.check_added_large_files import find_large_added_files
 from pre_commit_hooks.check_added_large_files import main
 from pre_commit_hooks.util import cmd_output
@@ -62,3 +66,40 @@ def test_integration(temp_git_dir):
 
         # Should fail with --maxkb
         assert main(argv=['--maxkb', '9', 'f.py']) == 1
+
+
+def has_gitlfs():
+    output = cmd_output('git', 'lfs', retcode=None, stderr=subprocess.STDOUT)
+    return 'git lfs status' in output
+
+
+xfailif_no_gitlfs = pytest.mark.xfail(
+    not has_gitlfs(), reason='This test requires git-lfs',
+)
+
+
+@xfailif_no_gitlfs
+def test_allows_gitlfs(temp_git_dir):  # pragma: no cover
+    with cwd(temp_git_dir):
+        # Work around https://github.com/github/git-lfs/issues/913
+        cmd_output('git', 'commit', '--allow-empty', '-m', 'foo')
+        cmd_output('git', 'lfs', 'install')
+        write_file('f.py', 'a' * 10000)
+        cmd_output('git', 'lfs', 'track', 'f.py')
+        cmd_output('git', 'add', '.')
+        # Should succeed
+        assert main(('--maxkb', '9', 'f.py')) == 0
+
+
+@xfailif_no_gitlfs
+def test_moves_with_gitlfs(temp_git_dir):  # pragma: no cover
+    with cwd(temp_git_dir):
+        cmd_output('git', 'lfs', 'install')
+        cmd_output('git', 'lfs', 'track', 'a.bin', 'b.bin')
+        # First add the file we're going to move
+        write_file('a.bin', 'a' * 10000)
+        cmd_output('git', 'add', '.')
+        cmd_output('git', 'commit', '-am', 'foo')
+        # Now move it and make sure the hook still succeeds
+        cmd_output('git', 'mv', 'a.bin', 'b.bin')
+        assert main(('--maxkb', '9', 'b.bin')) == 0