Merge pull request #3 from pre-commit/master

get up to date with upstream

pre_commit_hooks/check_added_large_files.py

@@ -6,7 +6,6 @@ from __future__ import unicode_literals
 import argparse
 import math
 import os
-import sys
 
 from pre_commit_hooks.util import added_files
 from pre_commit_hooks.util import CalledProcessError
@@ -16,7 +15,7 @@ from pre_commit_hooks.util import cmd_output
 def lfs_files():
     try:  # pragma: no cover (no git-lfs)
         lines = cmd_output('git', 'lfs', 'status', '--porcelain').splitlines()
-    except CalledProcessError:
+    except CalledProcessError:  # pragma: no cover (with git-lfs)
         lines = []
 
     modes_and_fileparts = [
@@ -49,8 +48,6 @@ def find_large_added_files(filenames, maxkb):
 
 
 def main(argv=None):
-    argv = argv if argv is not None else sys.argv[1:]
-
     parser = argparse.ArgumentParser()
     parser.add_argument(
         'filenames', nargs='*',

pre_commit_hooks/check_merge_conflict.py

@@ -2,7 +2,6 @@ from __future__ import print_function
 
 import argparse
 import os.path
-import sys
 
 CONFLICT_PATTERNS = [
     b'<<<<<<< ',
@@ -41,5 +40,6 @@ def detect_merge_conflict(argv=None):
 
     return retcode
 
+
 if __name__ == '__main__':
-    sys.exit(detect_merge_conflict())
+    exit(detect_merge_conflict())

pre_commit_hooks/detect_aws_credentials.py

@@ -7,65 +7,126 @@ import os
 from six.moves import configparser
 
 
-def get_your_keys(credentials_file):
-    """reads the secret keys in your credentials file in order to be able to
-    look for them in the submitted code.
+def get_aws_credential_files_from_env():
+    """Extract credential file paths from environment variables."""
+    files = set()
+    for env_var in (
+        'AWS_CONFIG_FILE', 'AWS_CREDENTIAL_FILE', 'AWS_SHARED_CREDENTIALS_FILE',
+        'BOTO_CONFIG'
+    ):
+        if env_var in os.environ:
+            files.add(os.environ[env_var])
+    return files
+
+
+def get_aws_secrets_from_env():
+    """Extract AWS secrets from environment variables."""
+    keys = set()
+    for env_var in (
+        'AWS_SECRET_ACCESS_KEY', 'AWS_SECURITY_TOKEN', 'AWS_SESSION_TOKEN'
+    ):
+        if env_var in os.environ:
+            keys.add(os.environ[env_var])
+    return keys
+
+
+def get_aws_secrets_from_file(credentials_file):
+    """Extract AWS secrets from configuration files.
+
+    Read an ini-style configuration file and return a set with all found AWS
+    secret access keys.
     """
     aws_credentials_file_path = os.path.expanduser(credentials_file)
     if not os.path.exists(aws_credentials_file_path):
-        return None
+        return set()
 
     parser = configparser.ConfigParser()
-    parser.read(aws_credentials_file_path)
+    try:
+        parser.read(aws_credentials_file_path)
+    except configparser.MissingSectionHeaderError:
+        return set()
 
     keys = set()
     for section in parser.sections():
-        keys.add(parser.get(section, 'aws_secret_access_key'))
+        for var in (
+            'aws_secret_access_key', 'aws_security_token',
+            'aws_session_token'
+        ):
+            try:
+                keys.add(parser.get(section, var))
+            except configparser.NoOptionError:
+                pass
     return keys
 
 
 def check_file_for_aws_keys(filenames, keys):
+    """Check if files contain AWS secrets.
+
+    Return a list of all files containing AWS secrets and keys found, with all
+    but the first four characters obfuscated to ease debugging.
+    """
     bad_files = []
 
     for filename in filenames:
         with open(filename, 'r') as content:
             text_body = content.read()
-            if any(key in text_body for key in keys):
-                # naively match the entire file, low chance of incorrect collision
-                bad_files.append(filename)
-
+            for key in keys:
+                # naively match the entire file, low chance of incorrect
+                # collision
+                if key in text_body:
+                    bad_files.append({
+                        'filename': filename, 'key': key[:4] + '*' * 28,
+                    })
     return bad_files
 
 
 def main(argv=None):
     parser = argparse.ArgumentParser()
-    parser.add_argument('filenames', nargs='*', help='Filenames to run')
+    parser.add_argument('filenames', nargs='+', help='Filenames to run')
     parser.add_argument(
         '--credentials-file',
-        default='~/.aws/credentials',
+        dest='credential_files',
+        action='append',
+        default=[
+            '~/.aws/config', '~/.aws/credentials', '/etc/boto.cfg', '~/.boto',
+        ],
         help=(
-            'location of aws credentials file from which to get the secret '
-            "keys we're looking for"
-        ),
+            'Location of additional AWS credential files from which to get '
+            'secret keys from'
+        )
     )
     args = parser.parse_args(argv)
-    keys = get_your_keys(args.credentials_file)
+
+    credential_files = set(args.credential_files)
+
+    # Add the credentials files configured via environment variables to the set
+    # of files to to gather AWS secrets from.
+    credential_files |= get_aws_credential_files_from_env()
+
+    keys = set()
+    for credential_file in credential_files:
+        keys |= get_aws_secrets_from_file(credential_file)
+
+    # Secrets might be part of environment variables, so add such secrets to
+    # the set of keys.
+    keys |= get_aws_secrets_from_env()
+
     if not keys:
         print(
-            'No aws keys were configured at {0}\n'
-            'Configure them with --credentials-file'.format(
-                args.credentials_file,
-            ),
+            'No AWS keys were found in the configured credential files and '
+            'environment variables.\nPlease ensure you have the correct '
+            'setting for --credentials-file'
         )
         return 2
 
     bad_filenames = check_file_for_aws_keys(args.filenames, keys)
     if bad_filenames:
         for bad_file in bad_filenames:
-            print('AWS secret key found: {0}'.format(bad_file))
+            print('AWS secret found in {filename}: {key}'.format(**bad_file))
         return 1
     else:
         return 0
 
+
 if __name__ == '__main__':
     exit(main())

pre_commit_hooks/detect_private_key.py

@@ -7,6 +7,7 @@ BLACKLIST = [
     b'BEGIN RSA PRIVATE KEY',
     b'BEGIN DSA PRIVATE KEY',
     b'BEGIN EC PRIVATE KEY',
+    b'BEGIN OPENSSH PRIVATE KEY',
 ]
 
 

pre_commit_hooks/fix_encoding_pragma.py

@@ -138,5 +138,6 @@ def main(argv=None):
 
     return retv
 
+
 if __name__ == "__main__":
     exit(main())

pre_commit_hooks/forbid_new_submodules.py

@@ -0,0 +1,33 @@
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from pre_commit_hooks.util import cmd_output
+
+
+def main(argv=None):
+    # `argv` is ignored, pre-commit will send us a list of files that we
+    # don't care about
+    added_diff = cmd_output(
+        'git', 'diff', '--staged', '--diff-filter=A', '--raw',
+    )
+    retv = 0
+    for line in added_diff.splitlines():
+        metadata, filename = line.split('\t', 1)
+        new_mode = metadata.split(' ')[1]
+        if new_mode == '160000':
+            print('{}: new submodule introduced'.format(filename))
+            retv = 1
+
+    if retv:
+        print()
+        print('This commit introduces new submodules.')
+        print('Did you unintentionally `git add .`?')
+        print('To fix: git rm {thesubmodule}  # no trailing slash')
+        print('Also check .gitmodules')
+
+    return retv
+
+
+if __name__ == '__main__':
+    exit(main())